diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs index 788c59e3..669e8fe4 100644 --- a/SabreTools.DatFiles/ItemDictionaryDB.cs +++ b/SabreTools.DatFiles/ItemDictionaryDB.cs @@ -1,20 +1,11 @@ -using System.Collections; -#if NET40_OR_GREATER || NETCOREAPP +#if NET40_OR_GREATER || NETCOREAPP using System.Collections.Concurrent; -#endif +#else using System.Collections.Generic; -using System.Linq; -#if NET40_OR_GREATER || NETCOREAPP -using System.Threading.Tasks; #endif using System.Xml.Serialization; using Newtonsoft.Json; -using SabreTools.Core; using SabreTools.DatItems; -using SabreTools.DatItems.Formats; -using SabreTools.Hashing; -using SabreTools.Logging; -using SabreTools.Matching; namespace SabreTools.DatFiles { @@ -22,760 +13,37 @@ namespace SabreTools.DatFiles /// Item dictionary with statistics, bucketing, and sorting /// [JsonObject("items"), XmlRoot("items")] - public class ItemDictionaryDB : IDictionary?> + public class ItemDictionaryDB { #region Private instance variables - /// - /// Determine the bucketing key for all items - /// - private ItemKey bucketedBy; - - /// - /// Determine merging type for all items - /// - private DedupeType mergedBy; - /// /// Internal dictionary for all items /// #if NET40_OR_GREATER || NETCOREAPP - private readonly ConcurrentDictionary items; + private readonly ConcurrentDictionary items = new ConcurrentDictionary(); #else - private readonly Dictionary items; + private readonly Dictionary items = []; #endif /// /// Internal dictionary for all machines /// #if NET40_OR_GREATER || NETCOREAPP - private readonly ConcurrentDictionary machines; + private readonly ConcurrentDictionary machines = new ConcurrentDictionary(); #else - private readonly Dictionary machines; + private readonly Dictionary machines = []; #endif /// /// Internal dictionary for item to machine mappings /// #if NET40_OR_GREATER || NETCOREAPP - private readonly ConcurrentDictionary itemToMachineMapping; + private readonly ConcurrentDictionary itemToMachineMapping = new ConcurrentDictionary(); #else - private readonly Dictionary machines; + private readonly Dictionary itemToMachineMapping = []; #endif - /// - /// Logging object - /// - private readonly Logger logger; - - #endregion - - #region Publically available fields - - #region Keys - - /// - /// Get the keys from the file dictionary - /// - /// List of the keys - [JsonIgnore, XmlIgnore] - public ICollection Keys - { - get { return items.Keys; } - } - - /// - /// Get the keys in sorted order from the file dictionary - /// - /// List of the keys in sorted order - [JsonIgnore, XmlIgnore] - public List SortedKeys - { - get - { - var keys = items.Keys.ToList(); - keys.Sort(new NaturalComparer()); - return keys; - } - } - - #endregion - - #region Statistics - - /// - /// DAT statistics - /// - [JsonIgnore, XmlIgnore] - public DatStatistics DatStatistics { get; } = new DatStatistics(); - - #endregion - - #endregion - - #region Accessors - - /// - /// Passthrough to access the file dictionary - /// - /// Key in the dictionary to reference - public ConcurrentList? this[string key] - { - get - { - // Explicit lock for some weird corner cases - lock (key) - { - // Ensure the key exists - EnsureKey(key); - - // Now return the value - return items[key]; - } - } - set - { - Remove(key); - if (value == null) - items[key] = null; - else - AddRange(key, value); - } - } - - /// - /// Add a value to the file dictionary - /// - /// Key in the dictionary to add to - /// Value to add to the dictionary - public void Add(string key, DatItem value) - { - // Explicit lock for some weird corner cases - lock (key) - { - // Ensure the key exists - EnsureKey(key); - - // If item is null, don't add it - if (value == null) - return; - - // Now add the value - items[key]!.Add(value); - - // Now update the statistics - DatStatistics.AddItemStatistics(value); - } - } - - /// - /// Add a range of values to the file dictionary - /// - /// Key in the dictionary to add to - /// Value to add to the dictionary - public void Add(string key, ConcurrentList? value) - { - AddRange(key, value); - } - - /// - /// Add a range of values to the file dictionary - /// - /// Key in the dictionary to add to - /// Value to add to the dictionary - public void AddRange(string key, ConcurrentList? value) - { - // Explicit lock for some weird corner cases - lock (key) - { - // If the value is null or empty, just return - if (value == null || !value.Any()) - return; - - // Ensure the key exists - EnsureKey(key); - - // Now add the value - items[key]!.AddRange(value); - - // Now update the statistics - foreach (DatItem item in value) - { - DatStatistics.AddItemStatistics(item); - } - } - } - - /// - /// Get if the file dictionary contains the key - /// - /// Key in the dictionary to check - /// True if the key exists, false otherwise - public bool ContainsKey(string key) - { - // If the key is null, we return false since keys can't be null - if (key == null) - return false; - - // Explicit lock for some weird corner cases - lock (key) - { - return items.ContainsKey(key); - } - } - - /// - /// Get if the file dictionary contains the key and value - /// - /// Key in the dictionary to check - /// Value in the dictionary to check - /// True if the key exists, false otherwise - public bool Contains(string key, DatItem value) - { - // If the key is null, we return false since keys can't be null - if (key == null) - return false; - - // Explicit lock for some weird corner cases - lock (key) - { - if (items.ContainsKey(key) && items[key] != null) - return items[key]!.Contains(value); - } - - return false; - } - - /// - /// Ensure the key exists in the items dictionary - /// - /// Key to ensure - public void EnsureKey(string key) - { - // If the key is missing from the dictionary, add it - if (!items.ContainsKey(key)) -#if NET40_OR_GREATER || NETCOREAPP - items.TryAdd(key, []); -#else - items[key] = []; -#endif - } - - /// - /// Get a list of filtered items for a given key - /// - /// Key in the dictionary to retrieve - public ConcurrentList FilteredItems(string key) - { - lock (key) - { - // Get the list, if possible - ConcurrentList? fi = items[key]; - if (fi == null) - return []; - - // Filter the list - return fi.Where(i => i != null) - .Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true) - .Where(i => i.GetFieldValue(DatItem.MachineKey) != null) - .ToConcurrentList(); - } - } - - /// - /// Remove a key from the file dictionary if it exists - /// - /// Key in the dictionary to remove - public bool Remove(string key) - { - // Explicit lock for some weird corner cases - lock (key) - { - // If the key doesn't exist, return - if (!ContainsKey(key) || items[key] == null) - return false; - - // Remove the statistics first - foreach (DatItem item in items[key]!) - { - DatStatistics.RemoveItemStatistics(item); - } - - // Remove the key from the dictionary -#if NET40_OR_GREATER || NETCOREAPP - return items.TryRemove(key, out _); -#else - return items.Remove(key); -#endif - } - } - - /// - /// Remove the first instance of a value from the file dictionary if it exists - /// - /// Key in the dictionary to remove from - /// Value to remove from the dictionary - public bool Remove(string key, DatItem value) - { - // Explicit lock for some weird corner cases - lock (key) - { - // If the key and value doesn't exist, return - if (!Contains(key, value) || items[key] == null) - return false; - - // Remove the statistics first - DatStatistics.RemoveItemStatistics(value); - - return items[key]!.Remove(value); - } - } - - /// - /// Reset a key from the file dictionary if it exists - /// - /// Key in the dictionary to reset - public bool Reset(string key) - { - // If the key doesn't exist, return - if (!ContainsKey(key) || items[key] == null) - return false; - - // Remove the statistics first - foreach (DatItem item in items[key]!) - { - DatStatistics.RemoveItemStatistics(item); - } - - // Remove the key from the dictionary - items[key] = []; - return true; - } - - /// - /// Override the internal ItemKey value - /// - /// - public void SetBucketedBy(ItemKey newBucket) - { - bucketedBy = newBucket; - } - - #endregion - - #region Constructors - - /// - /// Generic constructor - /// - public ItemDictionaryDB() - { - bucketedBy = ItemKey.NULL; - mergedBy = DedupeType.None; -#if NET40_OR_GREATER || NETCOREAPP - items = new ConcurrentDictionary?>(); - machines = new ConcurrentDictionary(); -#else - items = new Dictionary?>(); - machines = new Dictionary(); -#endif - logger = new Logger(this); - } - - #endregion - - #region Custom Functionality - - /// - /// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method - /// - /// ItemKey enum representing how to bucket the individual items - /// Dedupe type that should be used - /// True if the key should be lowercased (default), false otherwise - /// True if games should only be compared on game and file name, false if system and source are counted - public void BucketBy(ItemKey bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true) - { - // If we have a situation where there's no dictionary or no keys at all, we skip -#if NET40_OR_GREATER || NETCOREAPP - if (items == null || items.IsEmpty) -#else - if (items == null || items.Count == 0) -#endif - return; - - // If the sorted type isn't the same, we want to sort the dictionary accordingly - if (bucketedBy != bucketBy && bucketBy != ItemKey.NULL) - { - logger.User($"Organizing roms by {bucketBy}"); - - // Set the sorted type - bucketedBy = bucketBy; - - // Reset the merged type since this might change the merge - mergedBy = DedupeType.None; - - // First do the initial sort of all of the roms inplace - List oldkeys = [.. Keys]; - -#if NET452_OR_GREATER || NETCOREAPP - Parallel.For(0, oldkeys.Count, Globals.ParallelOptions, k => -#elif NET40_OR_GREATER - Parallel.For(0, oldkeys.Count, k => -#else - for (int k = 0; k < oldkeys.Count; k++) -#endif - { - string key = oldkeys[k]; - if (this[key] == null) - Remove(key); - - // Now add each of the roms to their respective keys - for (int i = 0; i < this[key]!.Count; i++) - { - DatItem item = this[key]![i]; - if (item == null) - continue; - - // We want to get the key most appropriate for the given sorting type - string newkey = item.GetKey(bucketBy, lower, norename); - - // If the key is different, move the item to the new key - if (newkey != key) - { - Add(newkey, item); - Remove(key, item); - i--; // This make sure that the pointer stays on the correct since one was removed - } - } - - // If the key is now empty, remove it - if (this[key]!.Count == 0) - Remove(key); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - - // If the merge type isn't the same, we want to merge the dictionary accordingly - if (mergedBy != dedupeType) - { - logger.User($"Deduping roms by {dedupeType}"); - - // Set the sorted type - mergedBy = dedupeType; - - List keys = [.. Keys]; -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(keys, Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(keys, key => -#else - foreach (var key in keys) -#endif - { - // Get the possibly unsorted list - ConcurrentList? sortedlist = this[key]?.ToConcurrentList(); - if (sortedlist == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - // Sort the list of items to be consistent - DatItem.Sort(ref sortedlist, false); - - // If we're merging the roms, do so - if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == ItemKey.Machine)) - sortedlist = DatItem.Merge(sortedlist); - - // Add the list back to the dictionary - Reset(key); - AddRange(key, sortedlist); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - // If the merge type is the same, we want to sort the dictionary to be consistent - else - { - List keys = [.. Keys]; -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(keys, Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(keys, key => -#else - foreach (var key in keys) -#endif - { - // Get the possibly unsorted list - ConcurrentList? sortedlist = this[key]; - - // Sort the list of items to be consistent - if (sortedlist != null) - DatItem.Sort(ref sortedlist, false); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - } - - /// - /// Remove any keys that have null or empty values - /// - public void ClearEmpty() - { - var keys = items.Keys.Where(k => k != null).ToList(); - foreach (string key in keys) - { - // If the key doesn't exist, skip - if (!items.ContainsKey(key)) - continue; - - // If the value is null, remove - else if (items[key] == null) -#if NET40_OR_GREATER || NETCOREAPP - items.TryRemove(key, out _); -#else - items.Remove(key); -#endif - - // If there are no non-blank items, remove - else if (!items[key]!.Any(i => i != null && i is not Blank)) -#if NET40_OR_GREATER || NETCOREAPP - items.TryRemove(key, out _); -#else - items.Remove(key); -#endif - } - } - - /// - /// Remove all items marked for removal - /// - public void ClearMarked() - { - var keys = items.Keys.ToList(); - foreach (string key in keys) - { - ConcurrentList? oldItemList = items[key]; - ConcurrentList? newItemList = oldItemList?.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)?.ToConcurrentList(); - - Remove(key); - AddRange(key, newItemList); - } - } - - /// - /// List all duplicates found in a DAT based on a DatItem - /// - /// Item to try to match - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// List of matched DatItem objects - public ConcurrentList GetDuplicates(DatItem datItem, bool sorted = false) - { - ConcurrentList output = []; - - // Check for an empty rom list first - if (DatStatistics.TotalCount == 0) - return output; - - // We want to get the proper key for the DatItem - string key = SortAndGetKey(datItem, sorted); - - // If the key doesn't exist, return the empty list - if (!ContainsKey(key)) - return output; - - // Try to find duplicates - ConcurrentList? roms = this[key]; - if (roms == null) - return output; - - ConcurrentList left = []; - for (int i = 0; i < roms.Count; i++) - { - DatItem other = roms[i]; - if (other.GetBoolFieldValue(DatItem.RemoveKey) == true) - continue; - - if (datItem.Equals(other)) - { - other.SetFieldValue(DatItem.RemoveKey, true); - output.Add(other); - } - else - { - left.Add(other); - } - } - - // Add back all roms with the proper flags - Remove(key); - AddRange(key, output); - AddRange(key, left); - - return output; - } - - /// - /// Check if a DAT contains the given DatItem - /// - /// Item to try to match - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// True if it contains the rom, false otherwise - public bool HasDuplicates(DatItem datItem, bool sorted = false) - { - // Check for an empty rom list first - if (DatStatistics.TotalCount == 0) - return false; - - // We want to get the proper key for the DatItem - string key = SortAndGetKey(datItem, sorted); - - // If the key doesn't exist, return the empty list - if (!ContainsKey(key)) - return false; - - // Try to find duplicates - ConcurrentList? roms = this[key]; - return roms?.Any(r => datItem.Equals(r)) == true; - } - - /// - /// Recalculate the statistics for the Dat - /// - public void RecalculateStats() - { - // Wipe out any stats already there - DatStatistics.ResetStatistics(); - - // If we have a blank Dat in any way, return - if (items == null) - return; - - // Loop through and add - foreach (string key in items.Keys) - { - ConcurrentList? datItems = items[key]; - if (datItems == null) - continue; - - foreach (DatItem item in datItems) - { - DatStatistics.AddItemStatistics(item); - } - } - } - - /// - /// Get the highest-order Field value that represents the statistics - /// - private ItemKey GetBestAvailable() - { - // Get the required counts - long diskCount = DatStatistics.GetItemCount(ItemType.Disk); - long mediaCount = DatStatistics.GetItemCount(ItemType.Media); - long romCount = DatStatistics.GetItemCount(ItemType.Rom); - long nodumpCount = DatStatistics.GetStatusCount(ItemStatus.Nodump); - - // If all items are supposed to have a SHA-512, we bucket by that - if (diskCount + mediaCount + romCount - nodumpCount == DatStatistics.GetHashCount(HashType.SHA512)) - return ItemKey.SHA512; - - // If all items are supposed to have a SHA-384, we bucket by that - else if (diskCount + mediaCount + romCount - nodumpCount == DatStatistics.GetHashCount(HashType.SHA384)) - return ItemKey.SHA384; - - // If all items are supposed to have a SHA-256, we bucket by that - else if (diskCount + mediaCount + romCount - nodumpCount == DatStatistics.GetHashCount(HashType.SHA256)) - return ItemKey.SHA256; - - // If all items are supposed to have a SHA-1, we bucket by that - else if (diskCount + mediaCount + romCount - nodumpCount == DatStatistics.GetHashCount(HashType.SHA1)) - return ItemKey.SHA1; - - // If all items are supposed to have a MD5, we bucket by that - else if (diskCount + mediaCount + romCount - nodumpCount == DatStatistics.GetHashCount(HashType.MD5)) - return ItemKey.MD5; - - // Otherwise, we bucket by CRC - else - return ItemKey.CRC; - } - - /// - /// Sort the input DAT and get the key to be used by the item - /// - /// Item to try to match - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// Key to try to use - private string SortAndGetKey(DatItem datItem, bool sorted = false) - { - // If we're not already sorted, take care of it - if (!sorted) - BucketBy(GetBestAvailable(), DedupeType.None); - - // Now that we have the sorted type, we get the proper key - return datItem.GetKey(bucketedBy); - } - - #endregion - - #region IDictionary Implementations - - public ICollection?> Values => ((IDictionary?>)items).Values; - - public int Count => ((ICollection?>>)items).Count; - - public bool IsReadOnly => ((ICollection?>>)items).IsReadOnly; - - public bool TryGetValue(string key, out ConcurrentList? value) - { - return ((IDictionary?>)items).TryGetValue(key, out value); - } - - public void Add(KeyValuePair?> item) - { - ((ICollection?>>)items).Add(item); - } - - public void Clear() - { - ((ICollection?>>)items).Clear(); - } - - public bool Contains(KeyValuePair?> item) - { - return ((ICollection?>>)items).Contains(item); - } - - public void CopyTo(KeyValuePair?>[] array, int arrayIndex) - { - ((ICollection?>>)items).CopyTo(array, arrayIndex); - } - - public bool Remove(KeyValuePair?> item) - { - return ((ICollection?>>)items).Remove(item); - } - - public IEnumerator?>> GetEnumerator() - { - return ((IEnumerable?>>)items).GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return ((IEnumerable)items).GetEnumerator(); - } - #endregion } }