#if NET40_OR_GREATER || NETCOREAPP using System.Collections.Concurrent; #endif using System.Collections.Generic; using System.Linq; #if NET40_OR_GREATER || NETCOREAPP using System.Threading.Tasks; #endif using System.Xml.Serialization; using Newtonsoft.Json; using SabreTools.Core; using SabreTools.DatItems; using SabreTools.DatItems.Formats; using SabreTools.Hashing; namespace SabreTools.DatFiles { /// /// Item dictionary with statistics, bucketing, and sorting /// [JsonObject("items"), XmlRoot("items")] public class ItemDictionaryDB { #region Private instance variables /// /// Internal dictionary for all items /// [JsonIgnore, XmlIgnore] #if NET40_OR_GREATER || NETCOREAPP private readonly ConcurrentDictionary _items = new ConcurrentDictionary(); #else private readonly Dictionary _items = []; #endif /// /// Current highest available item index /// [JsonIgnore, XmlIgnore] private long _itemIndex = 0; /// /// Internal dictionary for all machines /// [JsonIgnore, XmlIgnore] #if NET40_OR_GREATER || NETCOREAPP private readonly ConcurrentDictionary _machines = new ConcurrentDictionary(); #else private readonly Dictionary _machines = []; #endif /// /// Current highest available machine index /// [JsonIgnore, XmlIgnore] private long _machineIndex = 0; /// /// Internal dictionary for item to machine mappings /// [JsonIgnore, XmlIgnore] #if NET40_OR_GREATER || NETCOREAPP private readonly ConcurrentDictionary _itemToMachineMapping = new ConcurrentDictionary(); #else private readonly Dictionary _itemToMachineMapping = []; #endif /// /// Internal dictionary representing the current buckets /// [JsonIgnore, XmlIgnore] #if NET40_OR_GREATER || NETCOREAPP private readonly ConcurrentDictionary> _buckets = new ConcurrentDictionary>(); #else private readonly Dictionary> _buckets = []; #endif /// /// Current bucketed by value /// private ItemKey _bucketedBy = ItemKey.NULL; #endregion #region Fields /// /// DAT statistics /// [JsonIgnore, XmlIgnore] public DatStatistics DatStatistics { get; } = new DatStatistics(); #endregion #region Accessors /// /// Add an item, returning the insert index /// public long AddItem(DatItem item) { _items[_itemIndex++] = item; DatStatistics.AddItemStatistics(item); return _itemIndex - 1; } /// /// Add a machine, returning the insert index /// public long AddMachine(Machine machine) { _machines[_machineIndex++] = machine; return _machineIndex - 1; } /// /// Get an item based on the index /// public DatItem? GetItemByIndex(long index) { if (!_items.ContainsKey(index)) return null; return _items[index]; } /// /// Get a machine based on the index /// public Machine? GetMachineByIndex(long index) { if (!_machines.ContainsKey(index)) return null; return _machines[index]; } /// /// Get the machine associated with an item index /// public Machine? GetMachineForItemByIndex(long itemIndex) { if (!_itemToMachineMapping.ContainsKey(itemIndex)) return null; long machineIndex = _itemToMachineMapping[itemIndex]; if (!_machines.ContainsKey(machineIndex)) return null; return _machines[machineIndex]; } /// /// Get the items associated with a machine index /// public DatItem[]? GetDatItemsForMachineByIndex(long machineIndex) { var itemIds = _itemToMachineMapping .Where(mapping => mapping.Value == machineIndex) .Select(mapping => mapping.Key); var datItems = new List(); foreach (long itemId in itemIds) { if (_items.ContainsKey(itemId)) datItems.Add(_items[itemId]); } return datItems.ToArray(); } /// /// Remove an item, returning if it could be removed /// public bool RemoveItem(long itemIndex) { if (!_items.ContainsKey(itemIndex)) return false; #if NET40_OR_GREATER || NETCOREAPP _items.TryRemove(itemIndex, out _); #else _items.Remove(itemIndex); #endif if (_itemToMachineMapping.ContainsKey(itemIndex)) #if NET40_OR_GREATER || NETCOREAPP _itemToMachineMapping.TryRemove(itemIndex, out _); #else _itemToMachineMapping.Remove(itemIndex); #endif return true; } /// /// Remove a machine, returning if it could be removed /// public bool RemoveMachine(long machineIndex) { if (!_machines.ContainsKey(machineIndex)) return false; #if NET40_OR_GREATER || NETCOREAPP _machines.TryRemove(machineIndex, out _); #else _machines.Remove(machineIndex); #endif var itemIds = _itemToMachineMapping .Where(mapping => mapping.Value == machineIndex) .Select(mapping => mapping.Key); foreach (long itemId in itemIds) { #if NET40_OR_GREATER || NETCOREAPP _itemToMachineMapping.TryRemove(itemId, out _); #else _itemToMachineMapping.Remove(itemId); #endif } return true; } #endregion #region Bucketing /// /// Update the bucketing dictionary /// /// ItemKey enum representing how to bucket the individual items /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted /// public void UpdateBucketBy(ItemKey bucketBy, bool lower = true, bool norename = true) { // If the bucketing value is the same if (bucketBy == _bucketedBy) return; // Reset the bucketing values _bucketedBy = bucketBy; _buckets.Clear(); // Get the current list of item indicies long[] itemIndicies = _items.Keys.ToArray(); #if NET452_OR_GREATER || NETCOREAPP Parallel.For(0, itemIndicies.Length, Globals.ParallelOptions, i => #elif NET40_OR_GREATER Parallel.For(0, itemIndicies.Length, i => #else for (int i = 0; i < itemIndicies.Length; i++) #endif { string? bucketKey = GetBucketKey(i, bucketBy); EnsureBucketingKey(bucketKey); _buckets[bucketKey].Add(i); #if NET40_OR_GREATER || NETCOREAPP }); #else } #endif } /// /// Get the bucketing key for a given item index /// private string GetBucketKey(long itemIndex, ItemKey bucketBy) { if (!_items.ContainsKey(itemIndex)) return string.Empty; var datItem = _items[itemIndex]; if (datItem == null) return string.Empty; if (!_itemToMachineMapping.ContainsKey(itemIndex)) return string.Empty; long machineIndex = _itemToMachineMapping[itemIndex]; if (!_machines.ContainsKey(machineIndex)) return string.Empty; var machine = _machines[machineIndex]; if (machine == null) return string.Empty; return bucketBy switch { ItemKey.Machine => machine.GetStringFieldValue(Models.Metadata.Machine.NameKey) ?? string.Empty, _ => GetBucketHashValue(datItem, bucketBy), }; } /// /// Get the hash value for a given item, if possible /// private static string GetBucketHashValue(DatItem datItem, ItemKey bucketBy) { return datItem switch { Disk disk => bucketBy switch { ItemKey.CRC => Constants.CRCZero, ItemKey.MD5 => disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key) ?? string.Empty, ItemKey.SHA1 => disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key) ?? string.Empty, ItemKey.SHA256 => Constants.SHA256Zero, ItemKey.SHA384 => Constants.SHA384Zero, ItemKey.SHA512 => Constants.SHA512Zero, ItemKey.SpamSum => Constants.SpamSumZero, _ => string.Empty, }, Media media => bucketBy switch { ItemKey.CRC => Constants.CRCZero, ItemKey.MD5 => media.GetStringFieldValue(Models.Metadata.Media.MD5Key) ?? string.Empty, ItemKey.SHA1 => media.GetStringFieldValue(Models.Metadata.Media.SHA1Key) ?? string.Empty, ItemKey.SHA256 => media.GetStringFieldValue(Models.Metadata.Media.SHA256Key) ?? string.Empty, ItemKey.SHA384 => Constants.SHA384Zero, ItemKey.SHA512 => Constants.SHA512Zero, ItemKey.SpamSum => media.GetStringFieldValue(Models.Metadata.Media.SpamSumKey) ?? string.Empty, _ => string.Empty, }, Rom rom => bucketBy switch { ItemKey.CRC => rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) ?? string.Empty, ItemKey.MD5 => rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key) ?? string.Empty, ItemKey.SHA1 => rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key) ?? string.Empty, ItemKey.SHA256 => rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key) ?? string.Empty, ItemKey.SHA384 => rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key) ?? string.Empty, ItemKey.SHA512 => rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key) ?? string.Empty, ItemKey.SpamSum => rom.GetStringFieldValue(Models.Metadata.Rom.SpamSumKey) ?? string.Empty, _ => string.Empty, }, _ => bucketBy switch { ItemKey.CRC => Constants.CRCZero, ItemKey.MD5 => Constants.MD5Zero, ItemKey.SHA1 => Constants.SHA1Zero, ItemKey.SHA256 => Constants.SHA256Zero, ItemKey.SHA384 => Constants.SHA384Zero, ItemKey.SHA512 => Constants.SHA512Zero, ItemKey.SpamSum => Constants.SpamSumZero, _ => string.Empty, }, }; } /// /// Ensure the key exists in the items dictionary /// private void EnsureBucketingKey(string key) { // If the key is missing from the dictionary, add it if (!_buckets.ContainsKey(key)) #if NET40_OR_GREATER || NETCOREAPP _buckets.TryAdd(key, []); #else _buckets[key] = []; #endif } #endregion } }