From 4f28ae7f61e64a20f978dbee2636e7bf542c836a Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Sun, 26 Jul 2020 22:34:45 -0700 Subject: [PATCH] Add and use ItemDictionary class --- RombaSharp/RombaSharp.Help.cs | 26 +- RombaSharp/RombaSharp.Helpers.cs | 4 +- SabreTools.Library/DatFiles/AttractMode.cs | 4 +- SabreTools.Library/DatFiles/ClrMamePro.cs | 4 +- SabreTools.Library/DatFiles/DatFile.cs | 728 +++--------------- SabreTools.Library/DatFiles/DatStats.cs | 22 +- SabreTools.Library/DatFiles/DosCenter.cs | 4 +- SabreTools.Library/DatFiles/EverdriveSmdb.cs | 4 +- SabreTools.Library/DatFiles/Filter.cs | 261 +++---- SabreTools.Library/DatFiles/Hashfile.cs | 4 +- SabreTools.Library/DatFiles/ItemDictionary.cs | 576 ++++++++++++++ SabreTools.Library/DatFiles/Json.cs | 4 +- SabreTools.Library/DatFiles/Listrom.cs | 4 +- SabreTools.Library/DatFiles/Listxml.cs | 4 +- SabreTools.Library/DatFiles/Logiqx.cs | 4 +- SabreTools.Library/DatFiles/Missfile.cs | 4 +- SabreTools.Library/DatFiles/OfflineList.cs | 6 +- SabreTools.Library/DatFiles/OpenMSX.cs | 4 +- SabreTools.Library/DatFiles/RomCenter.cs | 4 +- SabreTools.Library/DatFiles/SabreDat.cs | 4 +- SabreTools.Library/DatFiles/SeparatedValue.cs | 4 +- SabreTools.Library/DatFiles/SoftwareList.cs | 12 +- 22 files changed, 879 insertions(+), 812 deletions(-) create mode 100644 SabreTools.Library/DatFiles/ItemDictionary.cs diff --git a/RombaSharp/RombaSharp.Help.cs b/RombaSharp/RombaSharp.Help.cs index 3e572dd3..1895a5ee 100644 --- a/RombaSharp/RombaSharp.Help.cs +++ b/RombaSharp/RombaSharp.Help.cs @@ -468,9 +468,9 @@ have a current entry in the DAT index."; string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; - foreach (string key in df.Keys) + foreach (string key in df.Items.Keys) { - List datItems = df[key]; + List datItems = df.Items[key]; foreach (Rom rom in datItems) { // If we care about if the file exists, check the databse first @@ -504,7 +504,7 @@ have a current entry in the DAT index."; } // Add to the Dat - need.Add(key, rom); + need.Items.Add(key, rom); } } // Otherwise, just add the file to the list @@ -532,7 +532,7 @@ have a current entry in the DAT index."; } // Add to the Dat - need.Add(key, rom); + need.Items.Add(key, rom); } } } @@ -1566,7 +1566,7 @@ contents of any changed dats."; // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually datroot.PopulateFromDir(_dats, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null); - datroot.BucketBy(BucketedBy.SHA1, DedupeType.None); + datroot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None); // Create a List of dat hashes in the database (SHA-1) List databaseDats = new List(); @@ -1585,9 +1585,9 @@ contents of any changed dats."; { sldr.Read(); string hash = sldr.GetString(0); - if (datroot.Contains(hash)) + if (datroot.Items.ContainsKey(hash)) { - datroot.Remove(hash); + datroot.Items.Remove(hash); databaseDats.Add(hash); } else if (!databaseDats.Contains(hash)) @@ -1596,7 +1596,7 @@ contents of any changed dats."; } } - datroot.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); + datroot.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); watch.Stop(); @@ -1605,9 +1605,9 @@ contents of any changed dats."; // Loop through the Dictionary and add all data watch.Start("Adding new DAT information"); - foreach (string key in datroot.Keys) + foreach (string key in datroot.Items.Keys) { - foreach (Rom value in datroot[key]) + foreach (Rom value in datroot.Items[key]) { AddDatToDatabase(value, dbc); } @@ -1695,7 +1695,7 @@ contents of any changed dats."; // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually depot.PopulateFromDir(depotname, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null); - depot.BucketBy(BucketedBy.SHA1, DedupeType.None); + depot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None); // Set the base queries to use string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; @@ -1706,10 +1706,10 @@ contents of any changed dats."; // Once we have both, check for any new files List dupehashes = new List(); - List keys = depot.Keys; + IEnumerable keys = depot.Items.Keys; foreach (string key in keys) { - List roms = depot[key]; + List roms = depot.Items[key]; foreach (Rom rom in roms) { if (hashes.Contains(rom.SHA1)) diff --git a/RombaSharp/RombaSharp.Helpers.cs b/RombaSharp/RombaSharp.Helpers.cs index bf6e367f..82902132 100644 --- a/RombaSharp/RombaSharp.Helpers.cs +++ b/RombaSharp/RombaSharp.Helpers.cs @@ -261,9 +261,9 @@ namespace RombaSharp // Loop through the parsed entries bool hasItems = false; - foreach (string romkey in tempdat.Keys) + foreach (string romkey in tempdat.Items.Keys) { - foreach (DatItem datItem in tempdat[romkey]) + foreach (DatItem datItem in tempdat.Items[romkey]) { Globals.Logger.Verbose($"Checking and adding file '{datItem.Name}'"); diff --git a/SabreTools.Library/DatFiles/AttractMode.cs b/SabreTools.Library/DatFiles/AttractMode.cs index d0e74752..43ea96ff 100644 --- a/SabreTools.Library/DatFiles/AttractMode.cs +++ b/SabreTools.Library/DatFiles/AttractMode.cs @@ -135,9 +135,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/ClrMamePro.cs b/SabreTools.Library/DatFiles/ClrMamePro.cs index 32985223..41824822 100644 --- a/SabreTools.Library/DatFiles/ClrMamePro.cs +++ b/SabreTools.Library/DatFiles/ClrMamePro.cs @@ -465,9 +465,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index 95dcfc43..a4316d90 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -16,6 +15,7 @@ using SabreTools.Library.Tools; using NaturalSort; [assembly: InternalsVisibleTo("SabreTools")] +[assembly: InternalsVisibleTo("RombaSharp")] namespace SabreTools.Library.DatFiles { /// @@ -29,20 +29,7 @@ namespace SabreTools.Library.DatFiles internal DatHeader DatHeader = new DatHeader(); // DatItems dictionary - internal ConcurrentDictionary> Items = new ConcurrentDictionary>(); - - // Internal statistical data - internal DatStats DatStats = new DatStats(); - - /// - /// Determine the bucketing key for all items - /// - private BucketedBy BucketedBy; - - /// - /// Determine merging type for all items - /// - private DedupeType MergedBy; + internal ItemDictionary Items = new ItemDictionary(); #endregion @@ -50,162 +37,6 @@ namespace SabreTools.Library.DatFiles #region Accessors - /// - /// Passthrough to access the file dictionary - /// - /// Key in the dictionary to reference - /// We don't want to allow direct setting of values because it bypasses the statistics - public List this[string key] - { - get - { - // Explicit lock for some weird corner cases - lock (key) - { - // Ensure the key exists - EnsureKey(key); - - // Now return the value - return Items[key]; - } - } - } - - /// - /// Add a value to the file dictionary - /// - /// Key in the dictionary to add to - /// Value to add to the dictionary - public void Add(string key, DatItem value) - { - // Explicit lock for some weird corner cases - lock (key) - { - // Ensure the key exists - EnsureKey(key); - - // If item is null, don't add it - if (value == null) - return; - - // Now add the value - Items[key].Add(value); - - // Now update the statistics - DatStats.AddItem(value); - } - } - - /// - /// Add a range of values to the file dictionary - /// - /// Key in the dictionary to add to - /// Value to add to the dictionary - public void AddRange(string key, List value) - { - // Explicit lock for some weird corner cases - lock (key) - { - // Ensure the key exists - EnsureKey(key); - - // Now add the value - Items[key].AddRange(value); - - // Now update the statistics - foreach (DatItem item in value) - { - DatStats.AddItem(item); - } - } - } - - /// - /// Get if the file dictionary contains the key - /// - /// Key in the dictionary to check - /// True if the key exists, false otherwise - public bool Contains(string key) - { - // If the key is null, we return false since keys can't be null - if (key == null) - return false; - - // Explicit lock for some weird corner cases - lock (key) - { - return Items.ContainsKey(key); - } - } - - /// - /// Get if the file dictionary contains the key and value - /// - /// Key in the dictionary to check - /// Value in the dictionary to check - /// True if the key exists, false otherwise - public bool Contains(string key, DatItem value) - { - // If the key is null, we return false since keys can't be null - if (key == null) - return false; - - // Explicit lock for some weird corner cases - lock (key) - { - if (Items.ContainsKey(key)) - return Items[key].Contains(value); - } - - return false; - } - - /// - /// Get the keys from the file dictionary - /// - /// List of the keys - public List Keys - { - get { return Items.Keys.ToList(); } - } - - /// - /// Remove a key from the file dictionary if it exists - /// - /// Key in the dictionary to remove - public void Remove(string key) - { - // If the key doesn't exist, return - if (!Contains(key)) - return; - - // Remove the statistics first - foreach (DatItem item in Items[key]) - { - DatStats.RemoveItem(item); - } - - // Remove the key from the dictionary - Items.TryRemove(key, out _); - } - - /// - /// Remove the first instance of a value from the file dictionary if it exists - /// - /// Key in the dictionary to remove from - /// Value to remove from the dictionary - public void Remove(string key, DatItem value) - { - // If the key and value doesn't exist, return - if (!Contains(key, value)) - return; - - // Remove the statistics first - DatStats.RemoveItem(value); - - Items[key].Remove(value); - } - /// /// Set the Date header value /// @@ -239,241 +70,6 @@ namespace SabreTools.Library.DatFiles DatHeader.Type = type; } - /// - /// Get the keys in sorted order from the file dictionary - /// - /// List of the keys in sorted order - public List SortedKeys - { - get - { - var keys = Items.Keys.ToList(); - keys.Sort(new NaturalComparer()); - return keys; - } - } - - /// - /// Ensure the key exists in the items dictionary - /// - /// Key to ensure - private void EnsureKey(string key) - { - // If the key is missing from the dictionary, add it - if (!Items.ContainsKey(key)) - Items.TryAdd(key, new List()); - } - - #endregion - - #region Bucketing - - /// - /// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method - /// - /// BucketedBy enum representing how to bucket the individual items - /// Dedupe type that should be used - /// True if the key should be lowercased (default), false otherwise - /// True if games should only be compared on game and file name, false if system and source are counted - public void BucketBy(BucketedBy bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true) - { - // If we have a situation where there's no dictionary or no keys at all, we skip - if (Items == null || Items.Count == 0) - return; - - // If the sorted type isn't the same, we want to sort the dictionary accordingly - if (this.BucketedBy != bucketBy) - { - Globals.Logger.User($"Organizing roms by {bucketBy}"); - - // Set the sorted type - this.BucketedBy = bucketBy; - - // Reset the merged type since this might change the merge - this.MergedBy = DedupeType.None; - - // First do the initial sort of all of the roms inplace - List oldkeys = Keys; - for (int k = 0; k < oldkeys.Count; k++) - { - string key = oldkeys[k]; - - // Get the unsorted current list - List items = this[key]; - - // Now add each of the roms to their respective keys - for (int i = 0; i < items.Count; i++) - { - DatItem item = items[i]; - if (item == null) - continue; - - // We want to get the key most appropriate for the given sorting type - string newkey = item.GetKey(bucketBy, lower, norename); - - // If the key is different, move the item to the new key - if (newkey != key) - { - Add(newkey, item); - Remove(key, item); - i--; // This make sure that the pointer stays on the correct since one was removed - } - } - - // If the key is now empty, remove it - if (this[key].Count == 0) - Remove(key); - } - } - - // If the merge type isn't the same, we want to merge the dictionary accordingly - if (this.MergedBy != dedupeType) - { - Globals.Logger.User($"Deduping roms by {dedupeType}"); - - // Set the sorted type - this.MergedBy = dedupeType; - - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - // Get the possibly unsorted list - List sortedlist = this[key]; - - // Sort the list of items to be consistent - DatItem.Sort(ref sortedlist, false); - - // If we're merging the roms, do so - if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == BucketedBy.Game)) - sortedlist = DatItem.Merge(sortedlist); - - // Add the list back to the dictionary - Remove(key); - AddRange(key, sortedlist); - }); - } - // If the merge type is the same, we want to sort the dictionary to be consistent - else - { - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - // Get the possibly unsorted list - List sortedlist = this[key]; - - // Sort the list of items to be consistent - DatItem.Sort(ref sortedlist, false); - }); - } - - // Now clean up all empty keys - CleanEmptyKeys(); - } - - /// - /// Clean out all empty keys in the dictionary - /// - private void CleanEmptyKeys() - { - List keys = Keys; - foreach (string key in keys) - { - if (this[key].Count == 0) - Remove(key); - } - } - - /// - /// Check if a DAT contains the given DatItem - /// - /// Item to try to match - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// True if it contains the rom, false otherwise - private bool HasDuplicates(DatItem datItem, bool sorted = false) - { - // Check for an empty rom list first - if (DatStats.Count == 0) - return false; - - // We want to get the proper key for the DatItem - string key = SortAndGetKey(datItem, sorted); - - // If the key doesn't exist, return the empty list - if (!Contains(key)) - return false; - - // Try to find duplicates - List roms = this[key]; - return roms.Any(r => datItem.Equals(r)); - } - - /// - /// List all duplicates found in a DAT based on a DatItem - /// - /// Item to try to match - /// True to mark matched roms for removal from the input, false otherwise (default) - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// List of matched DatItem objects - private List GetDuplicates(DatItem datItem, bool remove = false, bool sorted = false) - { - List output = new List(); - - // Check for an empty rom list first - if (DatStats.Count == 0) - return output; - - // We want to get the proper key for the DatItem - string key = SortAndGetKey(datItem, sorted); - - // If the key doesn't exist, return the empty list - if (!Contains(key)) - return output; - - // Try to find duplicates - List roms = this[key]; - List left = new List(); - for (int i = 0; i < roms.Count; i++) - { - DatItem other = roms[i]; - - if (datItem.Equals(other)) - { - other.Remove = true; - output.Add(other); - } - else - { - left.Add(other); - } - } - - // If we're in removal mode, add back all roms with the proper flags - if (remove) - { - Remove(key); - AddRange(key, output); - AddRange(key, left); - } - - return output; - } - - /// - /// Sort the input DAT and get the key to be used by the item - /// - /// Item to try to match - /// True if the DAT is already sorted accordingly, false otherwise (default) - /// Key to try to use - private string SortAndGetKey(DatItem datItem, bool sorted = false) - { - // If we're not already sorted, take care of it - if (!sorted) - BucketBy(DatStats.GetBestAvailable(), DedupeType.None); - - // Now that we have the sorted type, we get the proper key - return datItem.GetKey(BucketedBy); - } - #endregion #region Constructors @@ -488,9 +84,6 @@ namespace SabreTools.Library.DatFiles { DatHeader = datFile.DatHeader; this.Items = datFile.Items; - this.BucketedBy = datFile.BucketedBy; - this.MergedBy = datFile.MergedBy; - this.DatStats = datFile.DatStats; } } @@ -607,20 +200,19 @@ namespace SabreTools.Library.DatFiles public void AddFromExisting(DatFile datFile, bool delete = false) { // Get the list of keys from the DAT - List keys = datFile.Keys; - foreach (string key in keys) + foreach (string key in datFile.Items.Keys) { // Add everything from the key to the internal DAT - AddRange(key, datFile[key]); + Items.AddRange(key, datFile.Items[key]); // Now remove the key from the source DAT if (delete) - datFile.Remove(key); + datFile.Items.Remove(key); } // Now remove the file dictionary from the source DAT if (delete) - datFile.DeleteDictionary(); + datFile.Items = null; } /// @@ -847,18 +439,17 @@ namespace SabreTools.Library.DatFiles if (updateFields.Intersect(datItemFields).Any()) { // For comparison's sake, we want to use CRC as the base bucketing - BucketBy(BucketedBy.CRC, DedupeType.Full); - intDat.BucketBy(BucketedBy.CRC, DedupeType.None); + Items.BucketBy(BucketedBy.CRC, DedupeType.Full); + intDat.Items.BucketBy(BucketedBy.CRC, DedupeType.None); // Then we do a hashwise comparison against the base DAT - List keys = intDat.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key => { - List datItems = intDat[key]; + List datItems = intDat.Items[key]; List newDatItems = new List(); foreach (DatItem datItem in datItems) { - List dupes = GetDuplicates(datItem, sorted: true); + List dupes = Items.GetDuplicates(datItem, sorted: true); DatItem newDatItem = datItem.Clone() as DatItem; // Cast versions of the new DatItem for use below @@ -1145,8 +736,8 @@ namespace SabreTools.Library.DatFiles } // Now add the new list to the key - intDat.Remove(key); - intDat.AddRange(key, newDatItems); + intDat.Items.Remove(key); + intDat.Items.AddRange(key, newDatItems); }); } @@ -1154,21 +745,20 @@ namespace SabreTools.Library.DatFiles if (updateFields.Intersect(machineFields).Any()) { // For comparison's sake, we want to use Machine Name as the base bucketing - BucketBy(BucketedBy.Game, DedupeType.Full); - intDat.BucketBy(BucketedBy.Game, DedupeType.None); + Items.BucketBy(BucketedBy.Game, DedupeType.Full); + intDat.Items.BucketBy(BucketedBy.Game, DedupeType.None); // Then we do a namewise comparison against the base DAT - List keys = intDat.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key => { - List datItems = intDat[key]; + List datItems = intDat.Items[key]; List newDatItems = new List(); foreach (DatItem datItem in datItems) { DatItem newDatItem = datItem.Clone() as DatItem; - if (Contains(key) && this[key].Count() > 0) + if (Items.ContainsKey(key) && Items[key].Count() > 0) { - var firstDupe = this[key][0]; + var firstDupe = Items[key][0]; if (updateFields.Contains(Field.MachineName)) newDatItem.MachineName = firstDupe.MachineName; @@ -1235,8 +825,8 @@ namespace SabreTools.Library.DatFiles } // Now add the new list to the key - intDat.Remove(key); - intDat.AddRange(key, newDatItems); + intDat.Items.Remove(key); + intDat.Items.AddRange(key, newDatItems); }); } @@ -1260,7 +850,7 @@ namespace SabreTools.Library.DatFiles private void DiffAgainst(List inputFileNames, string outDir, bool inplace) { // For comparison's sake, we want to use CRC as the base ordering - BucketBy(BucketedBy.CRC, DedupeType.Full); + Items.BucketBy(BucketedBy.CRC, DedupeType.Full); // Now we want to compare each input DAT against the base foreach (string path in inputFileNames) @@ -1272,23 +862,22 @@ namespace SabreTools.Library.DatFiles intDat.Parse(path, 1, keep: true); // For comparison's sake, we want to use CRC as the base bucketing - intDat.BucketBy(BucketedBy.CRC, DedupeType.Full); + intDat.Items.BucketBy(BucketedBy.CRC, DedupeType.Full); // Then we do a hashwise comparison against the base DAT - List keys = intDat.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key => { - List datItems = intDat[key]; + List datItems = intDat.Items[key]; List keepDatItems = new List(); foreach (DatItem datItem in datItems) { - if (!HasDuplicates(datItem, true)) + if (!Items.HasDuplicates(datItem, true)) keepDatItems.Add(datItem); } // Now add the new list to the key - intDat.Remove(key); - intDat.AddRange(key, keepDatItems); + intDat.Items.Remove(key); + intDat.Items.AddRange(key, keepDatItems); }); // Determine the output path for the DAT @@ -1337,7 +926,7 @@ namespace SabreTools.Library.DatFiles diffData.DatHeader.Description += innerpost; } - diffData.ResetDictionary(); + diffData.Items = new ItemDictionary(); outDatsArray[j] = diffData; }); @@ -1345,15 +934,14 @@ namespace SabreTools.Library.DatFiles watch.Stop(); // Then, ensure that the internal dat can be bucketed in the best possible way - BucketBy(BucketedBy.CRC, DedupeType.None); + Items.BucketBy(BucketedBy.CRC, DedupeType.None); // Now, loop through the dictionary and populate the correct DATs watch.Start("Populating all output DATs"); - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(this[key]); + List items = DatItem.Merge(Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -1368,7 +956,7 @@ namespace SabreTools.Library.DatFiles continue; } - outDats[item.IndexId].Add(key, item); + outDats[item.IndexId].Items.Add(key, item); } }); @@ -1421,7 +1009,7 @@ namespace SabreTools.Library.DatFiles outerDiffData.DatHeader.FileName += post; outerDiffData.DatHeader.Name += post; outerDiffData.DatHeader.Description += post; - outerDiffData.ResetDictionary(); + outerDiffData.Items = new ItemDictionary(); } // Have External dupes @@ -1432,7 +1020,7 @@ namespace SabreTools.Library.DatFiles dupeData.DatHeader.FileName += post; dupeData.DatHeader.Name += post; dupeData.DatHeader.Description += post; - dupeData.ResetDictionary(); + dupeData.Items = new ItemDictionary(); } // Create a list of DatData objects representing individual output files @@ -1450,7 +1038,7 @@ namespace SabreTools.Library.DatFiles diffData.DatHeader.FileName += innerpost; diffData.DatHeader.Name += innerpost; diffData.DatHeader.Description += innerpost; - diffData.ResetDictionary(); + diffData.Items = new ItemDictionary(); outDatsArray[j] = diffData; }); @@ -1462,10 +1050,9 @@ namespace SabreTools.Library.DatFiles // Now, loop through the dictionary and populate the correct DATs watch.Start("Populating all output DATs"); - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(this[key]); + List items = DatItem.Merge(Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -1481,7 +1068,7 @@ namespace SabreTools.Library.DatFiles { // Individual DATs that are output if (diff.HasFlag(UpdateMode.DiffIndividualsOnly)) - outDats[item.IndexId].Add(key, item); + outDats[item.IndexId].Items.Add(key, item); // Merged no-duplicates DAT if (diff.HasFlag(UpdateMode.DiffNoDupesOnly)) @@ -1489,7 +1076,7 @@ namespace SabreTools.Library.DatFiles DatItem newrom = item.Clone() as DatItem; newrom.MachineName += $" ({Path.GetFileNameWithoutExtension(inputs[item.IndexId].Split('¬')[0])})"; - outerDiffData.Add(key, newrom); + outerDiffData.Items.Add(key, newrom); } } } @@ -1502,7 +1089,7 @@ namespace SabreTools.Library.DatFiles DatItem newrom = item.Clone() as DatItem; newrom.MachineName += $" ({Path.GetFileNameWithoutExtension(inputs[item.IndexId].Split('¬')[0])})"; - dupeData.Add(key, newrom); + dupeData.Items.Add(key, newrom); } } } @@ -1546,10 +1133,9 @@ namespace SabreTools.Library.DatFiles // If we're in SuperDAT mode, prefix all games with their respective DATs if (DatHeader.Type == "SuperDAT") { - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = this[key].ToList(); + List items = Items[key].ToList(); List newItems = new List(); foreach (DatItem item in items) { @@ -1566,8 +1152,8 @@ namespace SabreTools.Library.DatFiles newItems.Add(newItem); } - Remove(key); - AddRange(key, newItems); + Items.Remove(key); + Items.AddRange(key, newItems); }); } @@ -1605,36 +1191,6 @@ namespace SabreTools.Library.DatFiles #endregion - #region Dictionary Manipulation - - /// - /// Delete the file dictionary - /// - private void DeleteDictionary() - { - Items = null; - this.BucketedBy = BucketedBy.Default; - this.MergedBy = DedupeType.None; - - // Reset statistics - DatStats.Reset(); - } - - /// - /// Reset the file dictionary - /// - private void ResetDictionary() - { - Items = new ConcurrentDictionary>(); - this.BucketedBy = BucketedBy.Default; - this.MergedBy = DedupeType.None; - - // Reset statistics - DatStats.Reset(); - } - - #endregion - #region Parsing /// @@ -1670,7 +1226,7 @@ namespace SabreTools.Library.DatFiles // If the output type isn't set already, get the internal output type DatHeader.DatFormat = (DatHeader.DatFormat == 0 ? filename.GetDatFormat() : DatHeader.DatFormat); - this.BucketedBy = BucketedBy.CRC; // Setting this because it can reduce issues later + Items.SetBucketedBy(BucketedBy.CRC); // Setting this because it can reduce issues later // Now parse the correct type of DAT try @@ -1797,7 +1353,7 @@ namespace SabreTools.Library.DatFiles // Get the key and add the file key = item.GetKey(BucketedBy.CRC); - Add(key, item); + Items.Add(key, item); return key; } @@ -1902,7 +1458,7 @@ namespace SabreTools.Library.DatFiles romname = romname.Trim(Path.DirectorySeparatorChar); Globals.Logger.Verbose($"Adding blank empty folder: {gamename}"); - this["null"].Add(new Rom(romname, gamename)); + Items["null"].Add(new Rom(romname, gamename)); }); } } @@ -1952,7 +1508,7 @@ namespace SabreTools.Library.DatFiles { // Add the list if it doesn't exist already Rom rom = new Rom(baseFile); - Add(rom.GetKey(BucketedBy.CRC), rom); + Items.Add(rom.GetKey(BucketedBy.CRC), rom); Globals.Logger.User($"File added: {Path.GetFileNameWithoutExtension(item)}{Environment.NewLine}"); } else @@ -2079,7 +1635,7 @@ namespace SabreTools.Library.DatFiles // Add the file information to the DAT string key = datItem.GetKey(BucketedBy.CRC); - Add(key, datItem); + Items.Add(key, datItem); Globals.Logger.User($"File added: {datItem.Name}{Environment.NewLine}"); } @@ -2187,7 +1743,7 @@ namespace SabreTools.Library.DatFiles #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit - if (DatStats.Count == 0 && !inverse) + if (Items.Statistics.Count == 0 && !inverse) { Globals.Logger.User("No entries were found to rebuild, exiting..."); return false; @@ -2272,11 +1828,10 @@ namespace SabreTools.Library.DatFiles return success; // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 - BucketBy(BucketedBy.SHA1, DedupeType.None); + Items.BucketBy(BucketedBy.SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild - List hashes = Keys; - foreach (string hash in hashes) + foreach (string hash in Items.Keys) { // Pre-empt any issues that could arise from string length if (hash.Length != Constants.SHA1Length) @@ -2312,7 +1867,7 @@ namespace SabreTools.Library.DatFiles // Otherwise, we rebuild that file to all locations that we need to bool usedInternally; - if (this[hash][0].ItemType == ItemType.Disk) + if (Items[hash][0].ItemType == ItemType.Disk) usedInternally = RebuildIndividualFile(new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, updateDat, false /* isZip */, headerToCheckAgainst); else usedInternally = RebuildIndividualFile(new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, updateDat, false /* isZip */, headerToCheckAgainst); @@ -2332,7 +1887,7 @@ namespace SabreTools.Library.DatFiles DatHeader.FileName = $"fixDAT_{DatHeader.FileName}"; DatHeader.Name = $"fixDAT_{DatHeader.Name}"; DatHeader.Description = $"fixDAT_{DatHeader.Description}"; - RemoveMarkedItems(); + Items.ClearMarked(); Write(outDir); } @@ -2368,7 +1923,7 @@ namespace SabreTools.Library.DatFiles #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit - if (DatStats.Count == 0 && !inverse) + if (Items.Statistics.Count == 0 && !inverse) { Globals.Logger.User("No entries were found to rebuild, exiting..."); return false; @@ -2469,7 +2024,7 @@ namespace SabreTools.Library.DatFiles DatHeader.FileName = $"fixDAT_{DatHeader.FileName}"; DatHeader.Name = $"fixDAT_{DatHeader.Name}"; DatHeader.Description = $"fixDAT_{DatHeader.Description}"; - RemoveMarkedItems(); + Items.ClearMarked(); Write(outDir); } @@ -2615,7 +2170,7 @@ namespace SabreTools.Library.DatFiles string sha1 = ((Rom)datItem).SHA1 ?? string.Empty; // Find if the file has duplicates in the DAT - List dupes = GetDuplicates(datItem, remove: updateDat); + List dupes = Items.GetDuplicates(datItem, remove: updateDat); bool hasDuplicates = dupes.Count > 0; // If either we have duplicates or we're filtering @@ -2775,7 +2330,7 @@ namespace SabreTools.Library.DatFiles Rom headerless = new Rom(transformStream.GetInfo(keepReadOpen: true)); // Find if the file has duplicates in the DAT - dupes = GetDuplicates(headerless, remove: updateDat); + dupes = Items.GetDuplicates(headerless, remove: updateDat); hasDuplicates = dupes.Count > 0; // If it has duplicates and we're not filtering, rebuild it @@ -2846,11 +2401,10 @@ namespace SabreTools.Library.DatFiles return success; // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 - BucketBy(BucketedBy.SHA1, DedupeType.None); + Items.BucketBy(BucketedBy.SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild - List hashes = Keys; - foreach (string hash in hashes) + foreach (string hash in Items.Keys) { // Pre-empt any issues that could arise from string length if (hash.Length != Constants.SHA1Length) @@ -2885,8 +2439,8 @@ namespace SabreTools.Library.DatFiles continue; // Now we want to remove all duplicates from the DAT - GetDuplicates(new Rom(fileinfo), remove: true) - .AddRange(GetDuplicates(new Disk(fileinfo), remove: true)); + Items.GetDuplicates(new Rom(fileinfo), remove: true) + .AddRange(Items.GetDuplicates(new Disk(fileinfo), remove: true)); } watch.Stop(); @@ -2895,7 +2449,7 @@ namespace SabreTools.Library.DatFiles DatHeader.FileName = $"fixDAT_{DatHeader.FileName}"; DatHeader.Name = $"fixDAT_{DatHeader.Name}"; DatHeader.Description = $"fixDAT_{DatHeader.Description}"; - RemoveMarkedItems(); + Items.ClearMarked(); Write(); return success; @@ -2927,7 +2481,7 @@ namespace SabreTools.Library.DatFiles // Setup the fixdat DatFile matched = Create(DatHeader); - matched.ResetDictionary(); + matched.Items = new ItemDictionary(); matched.DatHeader.FileName = $"fixDat_{matched.DatHeader.FileName}"; matched.DatHeader.Name = $"fixDat_{matched.DatHeader.Name}"; matched.DatHeader.Description = $"fixDat_{matched.DatHeader.Description}"; @@ -2937,18 +2491,18 @@ namespace SabreTools.Library.DatFiles if (hashOnly) { // First we need to bucket and dedupe by hash to get duplicates - BucketBy(BucketedBy.CRC, DedupeType.Full); + Items.BucketBy(BucketedBy.CRC, DedupeType.Full); // Then follow the same tactics as before - foreach (string key in Keys) + foreach (string key in Items.Keys) { - List roms = this[key]; + List roms = Items[key]; foreach (DatItem rom in roms) { if (rom.IndexId == 99) { if (rom.ItemType == ItemType.Disk || rom.ItemType == ItemType.Rom) - matched.Add(((Disk)rom).SHA1, rom); + matched.Items.Add(((Disk)rom).SHA1, rom); } } } @@ -2956,41 +2510,25 @@ namespace SabreTools.Library.DatFiles // If we are checking full names, get only files found in directory else { - foreach (string key in Keys) + foreach (string key in Items.Keys) { - List roms = this[key]; + List roms = Items[key]; List newroms = DatItem.Merge(roms); foreach (Rom rom in newroms) { if (rom.IndexId == 99) - matched.Add($"{rom.Size}-{rom.CRC}", rom); + matched.Items.Add($"{rom.Size}-{rom.CRC}", rom); } } } // Now output the fixdat to the main folder - RemoveMarkedItems(); + Items.ClearMarked(); success &= matched.Write(stats: true); return success; } - /// - /// Remove all items marked for removal from the DAT - /// - private void RemoveMarkedItems() - { - List keys = Keys; - foreach (string key in keys) - { - List items = this[key]; - List newItems = items.Where(i => !i.Remove).ToList(); - - Remove(key); - AddRange(key, newItems); - } - } - #endregion // TODO: Implement Level split @@ -3046,7 +2584,7 @@ namespace SabreTools.Library.DatFiles // Now re-empty the DAT to make room for the next one DatFormat tempFormat = DatHeader.DatFormat; DatHeader = new DatHeader(); - ResetDictionary(); + Items = new ItemDictionary(); DatHeader.DatFormat = tempFormat; } } @@ -3061,7 +2599,7 @@ namespace SabreTools.Library.DatFiles private bool SplitByExtension(string outDir, List extA, List extB) { // If roms is empty, return false - if (DatStats.Count == 0) + if (Items.Statistics.Count == 0) return false; // Make sure all of the extensions don't have a dot at the beginning @@ -3083,24 +2621,23 @@ namespace SabreTools.Library.DatFiles datdataB.DatHeader.Description += $" ({newExtBString})"; // Now separate the roms accordingly - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = this[key]; + List items = Items[key]; foreach (DatItem item in items) { if (newExtA.Contains(PathExtensions.GetNormalizedExtension(item.Name))) { - datdataA.Add(key, item); + datdataA.Items.Add(key, item); } else if (newExtB.Contains(PathExtensions.GetNormalizedExtension(item.Name))) { - datdataB.Add(key, item); + datdataB.Items.Add(key, item); } else { - datdataA.Add(key, item); - datdataB.Add(key, item); + datdataA.Items.Add(key, item); + datdataB.Items.Add(key, item); } } }); @@ -3170,10 +2707,9 @@ namespace SabreTools.Library.DatFiles other.DatHeader.Description += " (Other)"; // Now populate each of the DAT objects in turn - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = this[key]; + List items = Items[key]; foreach (DatItem item in items) { // If the file is not a Rom or Disk, continue @@ -3184,54 +2720,54 @@ namespace SabreTools.Library.DatFiles if ((item.ItemType == ItemType.Rom && ((Rom)item).ItemStatus == ItemStatus.Nodump) || (item.ItemType == ItemType.Disk && ((Disk)item).ItemStatus == ItemStatus.Nodump)) { - nodump.Add(key, item); + nodump.Items.Add(key, item); } // If the file has a SHA-512 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).SHA512)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).SHA512))) { - sha512.Add(key, item); + sha512.Items.Add(key, item); } // If the file has a SHA-384 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).SHA384)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).SHA384))) { - sha384.Add(key, item); + sha384.Items.Add(key, item); } // If the file has a SHA-256 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).SHA256)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).SHA256))) { - sha256.Add(key, item); + sha256.Items.Add(key, item); } // If the file has a SHA-1 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).SHA1)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).SHA1))) { - sha1.Add(key, item); + sha1.Items.Add(key, item); } #if NET_FRAMEWORK // If the file has a RIPEMD160 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).RIPEMD160)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).RIPEMD160))) { - ripemd160.Add(key, item); + ripemd160.Items.Add(key, item); } #endif // If the file has an MD5 else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).MD5)) || (item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace(((Disk)item).MD5))) { - md5.Add(key, item); + md5.Items.Add(key, item); } // If the file has a CRC else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace(((Rom)item).CRC))) { - crc.Add(key, item); + crc.Items.Add(key, item); } else { - other.Add(key, item); + other.Items.Add(key, item); } } }); @@ -3263,14 +2799,14 @@ namespace SabreTools.Library.DatFiles private bool SplitByLevel(string outDir, bool shortname, bool basedat) { // First, bucket by games so that we can do the right thing - BucketBy(BucketedBy.Game, DedupeType.None, lower: false, norename: true); + Items.BucketBy(BucketedBy.Game, DedupeType.None, lower: false, norename: true); // Create a temporary DAT to add things to DatFile tempDat = Create(DatHeader); tempDat.DatHeader.Name = null; // Sort the input keys - List keys = Keys; + List keys = Items.Keys.ToList(); keys.Sort(SplitByLevelSort); // Then, we loop over the games @@ -3285,12 +2821,12 @@ namespace SabreTools.Library.DatFiles } // Clean the input list and set all games to be pathless - List items = this[key]; + List items = Items[key]; items.ForEach(item => item.MachineName = Path.GetFileName(item.MachineName)); items.ForEach(item => item.MachineDescription = Path.GetFileName(item.MachineDescription)); // Now add the game to the output DAT - tempDat.AddRange(key, items); + tempDat.Items.AddRange(key, items); // Then set the DAT name to be the parent directory name tempDat.DatHeader.Name = Path.GetDirectoryName(key); @@ -3369,23 +2905,22 @@ namespace SabreTools.Library.DatFiles greaterEqualDat.DatHeader.Description += $" (equal-greater than {radix})"; // Now populate each of the DAT objects in turn - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = this[key]; + List items = Items[key]; foreach (DatItem item in items) { // If the file is not a Rom, it automatically goes in the "lesser" dat if (item.ItemType != ItemType.Rom) - lessDat.Add(key, item); + lessDat.Items.Add(key, item); // If the file is a Rom and less than the radix, put it in the "lesser" dat else if (item.ItemType == ItemType.Rom && ((Rom)item).Size < radix) - lessDat.Add(key, item); + lessDat.Items.Add(key, item); // If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat else if (item.ItemType == ItemType.Rom && ((Rom)item).Size >= radix) - greaterEqualDat.Add(key, item); + greaterEqualDat.Items.Add(key, item); } }); @@ -3424,23 +2959,22 @@ namespace SabreTools.Library.DatFiles sampledat.DatHeader.Description += " (Sample)"; // Now populate each of the DAT objects in turn - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => { - List items = this[key]; + List items = Items[key]; foreach (DatItem item in items) { // If the file is a Rom if (item.ItemType == ItemType.Rom) - romdat.Add(key, item); + romdat.Items.Add(key, item); // If the file is a Disk else if (item.ItemType == ItemType.Disk) - diskdat.Add(key, item); + diskdat.Items.Add(key, item); // If the file is a Sample else if (item.ItemType == ItemType.Sample) - sampledat.Add(key, item); + sampledat.Items.Add(key, item); } }); @@ -3456,34 +2990,6 @@ namespace SabreTools.Library.DatFiles #endregion - #region Statistics - - /// - /// Recalculate the statistics for the Dat - /// - private void RecalculateStats() - { - // Wipe out any stats already there - DatStats.Reset(); - - // If we have a blank Dat in any way, return - if (this == null || DatStats.Count == 0) - return; - - // Loop through and add - List keys = Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - List items = this[key]; - foreach (DatItem item in items) - { - DatStats.AddItem(item); - } - }); - } - - #endregion - #region Writing /// @@ -3498,7 +3004,7 @@ namespace SabreTools.Library.DatFiles public bool Write(string outDir = null, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) { // If there's nothing there, abort - if (DatStats.Count == 0) + if (Items.Statistics.Count == 0) { Globals.Logger.User("There were no items to write out!"); return false; @@ -3551,30 +3057,30 @@ namespace SabreTools.Library.DatFiles // Output initial statistics, for kicks if (stats) { - if (DatStats.RomCount + DatStats.DiskCount == 0) - RecalculateStats(); + if (Items.Statistics.RomCount + Items.Statistics.DiskCount == 0) + Items.RecalculateStats(); - BucketBy(BucketedBy.Game, DedupeType.None, norename: true); + Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); // TODO: How can the size be negative when dealing with Int64? - if (DatStats.TotalSize < 0) - DatStats.TotalSize = Int64.MaxValue + DatStats.TotalSize; + if (Items.Statistics.TotalSize < 0) + Items.Statistics.TotalSize = Int64.MaxValue + Items.Statistics.TotalSize; var consoleOutput = BaseReport.Create(StatReportFormat.None, null, true, true); - consoleOutput.ReplaceStatistics(DatHeader.FileName, Keys.Count(), DatStats); + consoleOutput.ReplaceStatistics(DatHeader.FileName, Items.Keys.Count(), Items.Statistics); } // Bucket and dedupe according to the flag if (DatHeader.DedupeRoms == DedupeType.Full) - BucketBy(BucketedBy.CRC, DatHeader.DedupeRoms, norename: norename); + Items.BucketBy(BucketedBy.CRC, DatHeader.DedupeRoms, norename: norename); else if (DatHeader.DedupeRoms == DedupeType.Game) - BucketBy(BucketedBy.Game, DatHeader.DedupeRoms, norename: norename); + Items.BucketBy(BucketedBy.Game, DatHeader.DedupeRoms, norename: norename); // Bucket roms by game name, if not already - BucketBy(BucketedBy.Game, DedupeType.None, norename: norename); + Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: norename); // Output the number of items we're going to be writing - Globals.Logger.User($"A total of {DatStats.Count} items will be written out to '{DatHeader.FileName}'"); + Globals.Logger.User($"A total of {Items.Statistics.Count} items will be written out to '{DatHeader.FileName}'"); // Get the outfile names Dictionary outfiles = DatHeader.CreateOutFileNames(outDir, overwrite); diff --git a/SabreTools.Library/DatFiles/DatStats.cs b/SabreTools.Library/DatFiles/DatStats.cs index 06850414..981eac27 100644 --- a/SabreTools.Library/DatFiles/DatStats.cs +++ b/SabreTools.Library/DatFiles/DatStats.cs @@ -411,8 +411,14 @@ namespace SabreTools.Library.DatFiles /// True if baddumps should be included in output, false otherwise /// True if nodumps should be included in output, false otherwise /// Set the statistics output format to use - public static void OutputStats(List inputs, string reportName, string outDir, bool single, - bool baddumpCol, bool nodumpCol, StatReportFormat statDatFormat) + public static void OutputStats( + List inputs, + string reportName, + string outDir, + bool single, + bool baddumpCol, + bool nodumpCol, + StatReportFormat statDatFormat) { // If there's no output format, set the default if (statDatFormat == StatReportFormat.None) @@ -480,23 +486,23 @@ namespace SabreTools.Library.DatFiles Globals.Logger.Verbose($"Beginning stat collection for '{file}'", false); List games = new List(); DatFile datdata = DatFile.CreateAndParse(file); - datdata.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); + datdata.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); // Output single DAT stats (if asked) Globals.Logger.User($"Adding stats for file '{file}'\n", false); if (single) { - reports.ForEach(report => report.ReplaceStatistics(datdata.DatHeader.FileName, datdata.Keys.Count, datdata.DatStats)); + reports.ForEach(report => report.ReplaceStatistics(datdata.DatHeader.FileName, datdata.Items.Keys.Count, datdata.Items.Statistics)); reports.ForEach(report => report.Write()); } // Add single DAT stats to dir - dirStats.AddStats(datdata.DatStats); - dirStats.GameCount += datdata.Keys.Count(); + dirStats.AddStats(datdata.Items.Statistics); + dirStats.GameCount += datdata.Items.Keys.Count(); // Add single DAT stats to totals - totalStats.AddStats(datdata.DatStats); - totalStats.GameCount += datdata.Keys.Count(); + totalStats.AddStats(datdata.Items.Statistics); + totalStats.GameCount += datdata.Items.Keys.Count(); // Make sure to assign the new directory lastdir = thisdir; diff --git a/SabreTools.Library/DatFiles/DosCenter.cs b/SabreTools.Library/DatFiles/DosCenter.cs index c5d02daa..39c0441e 100644 --- a/SabreTools.Library/DatFiles/DosCenter.cs +++ b/SabreTools.Library/DatFiles/DosCenter.cs @@ -291,9 +291,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/EverdriveSmdb.cs b/SabreTools.Library/DatFiles/EverdriveSmdb.cs index aabf13c5..a91ae4af 100644 --- a/SabreTools.Library/DatFiles/EverdriveSmdb.cs +++ b/SabreTools.Library/DatFiles/EverdriveSmdb.cs @@ -111,9 +111,9 @@ namespace SabreTools.Library.DatFiles }; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/Filter.cs b/SabreTools.Library/DatFiles/Filter.cs index a8c056fe..d9f5eb9a 100644 --- a/SabreTools.Library/DatFiles/Filter.cs +++ b/SabreTools.Library/DatFiles/Filter.cs @@ -781,25 +781,24 @@ namespace SabreTools.Library.DatFiles // We remove any blanks, if we aren't supposed to have any if (!datFile.DatHeader.KeepEmptyGames) { - foreach (string key in datFile.Keys) + foreach (string key in datFile.Items.Keys) { - List items = datFile[key]; + List items = datFile.Items[key]; if (items == null) continue; List newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList(); - datFile.Remove(key); - datFile.AddRange(key, newitems); + datFile.Items.Remove(key); + datFile.Items.AddRange(key, newitems); } } // Loop over every key in the dictionary - List keys = datFile.Keys; - foreach (string key in keys) + foreach (string key in datFile.Items.Keys) { // For every item in the current key - List items = datFile[key]; + List items = datFile.Items[key]; List newitems = new List(); foreach (DatItem item in items) { @@ -843,8 +842,8 @@ namespace SabreTools.Library.DatFiles } } - datFile.Remove(key); - datFile.AddRange(key, newitems); + datFile.Items.Remove(key); + datFile.Items.AddRange(key, newitems); } // If we are removing scene dates, do that now @@ -890,25 +889,24 @@ namespace SabreTools.Library.DatFiles // We remove any blanks, if we aren't supposed to have any if (!outDat.DatHeader.KeepEmptyGames) { - foreach (string key in outDat.Keys) + foreach (string key in outDat.Items.Keys) { - List items = outDat[key]; + List items = outDat.Items[key]; if (items == null) continue; List newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList(); - outDat.Remove(key); - outDat.AddRange(key, newitems); + outDat.Items.Remove(key); + outDat.Items.AddRange(key, newitems); } } // Loop over every key in the dictionary - List keys = datFile.Keys; - foreach (string key in keys) + foreach (string key in datFile.Items.Keys) { // For every item in the current key - List items = datFile[key]; + List items = datFile.Items[key]; List newitems = new List(); foreach (DatItem item in items) { @@ -955,7 +953,7 @@ namespace SabreTools.Library.DatFiles } } - outDat.AddRange(key, newitems); + outDat.Items.AddRange(key, newitems); } // If we are removing scene dates, do that now @@ -1415,7 +1413,7 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("Creating device non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); + datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information while (AddRomsFromDevices(datFile, false, false)) ; @@ -1435,7 +1433,7 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("Creating fully non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); + datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information while (AddRomsFromDevices(datFile, true, true)) ; @@ -1459,7 +1457,7 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("Creating merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); + datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information AddRomsFromChildren(datFile); @@ -1482,7 +1480,7 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("Creating non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); + datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information AddRomsFromParent(datFile); @@ -1505,7 +1503,7 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("Creating split sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); + datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information RemoveRomsFromChild(datFile); @@ -1524,37 +1522,35 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void AddRomsFromBios(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue - if (datFile[game].Count == 0) + if (datFile.Items[game].Count == 0) continue; // Determine if the game has a parent or not string parent = null; - if (!string.IsNullOrWhiteSpace(datFile[game][0].RomOf)) - parent = datFile[game][0].RomOf; + if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].RomOf)) + parent = datFile.Items[game][0].RomOf; // If the parent doesnt exist, we want to continue if (string.IsNullOrWhiteSpace(parent)) continue; // If the parent doesn't have any items, we want to continue - if (datFile[parent].Count == 0) + if (datFile.Items[parent].Count == 0) continue; // If the parent exists and has items, we copy the items from the parent to the current game - DatItem copyFrom = datFile[game][0]; - List parentItems = datFile[parent]; + DatItem copyFrom = datFile.Items[game][0]; + List parentItems = datFile.Items[parent]; foreach (DatItem item in parentItems) { DatItem datItem = (DatItem)item.Clone(); datItem.CopyMachineInformation(copyFrom); - if (datFile[game].Where(i => i.Name == datItem.Name).Count() == 0 && !datFile[game].Contains(datItem)) - datFile.Add(game, datItem); + if (datFile.Items[game].Where(i => i.Name == datItem.Name).Count() == 0 && !datFile.Items[game].Contains(datItem)) + datFile.Items.Add(game, datItem); } } } @@ -1568,49 +1564,47 @@ namespace SabreTools.Library.DatFiles private bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool slotoptions = false) { bool foundnew = false; - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game doesn't have items, we continue - if (datFile[game] == null || datFile[game].Count == 0) + if (datFile.Items[game] == null || datFile.Items[game].Count == 0) continue; // If the game (is/is not) a bios, we want to continue - if (dev ^ (datFile[game][0].MachineType.HasFlag(MachineType.Device))) + if (dev ^ (datFile.Items[game][0].MachineType.HasFlag(MachineType.Device))) continue; // If the game has no devices, we continue - if (datFile[game][0].Devices == null - || datFile[game][0].Devices.Count == 0 - || (slotoptions && datFile[game][0].SlotOptions == null) - || (slotoptions && datFile[game][0].SlotOptions.Count == 0)) + if (datFile.Items[game][0].Devices == null + || datFile.Items[game][0].Devices.Count == 0 + || (slotoptions && datFile.Items[game][0].SlotOptions == null) + || (slotoptions && datFile.Items[game][0].SlotOptions.Count == 0)) { continue; } // Determine if the game has any devices or not - List devices = datFile[game][0].Devices; + List devices = datFile.Items[game][0].Devices; List newdevs = new List(); foreach (string device in devices) { // If the device doesn't exist then we continue - if (datFile[device].Count == 0) + if (datFile.Items[device].Count == 0) continue; // Otherwise, copy the items from the device to the current game - DatItem copyFrom = datFile[game][0]; - List devItems = datFile[device]; + DatItem copyFrom = datFile.Items[game][0]; + List devItems = datFile.Items[device]; foreach (DatItem item in devItems) { DatItem datItem = (DatItem)item.Clone(); newdevs.AddRange(datItem.Devices ?? new List()); datItem.CopyMachineInformation(copyFrom); - if (datFile[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) + if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) { foundnew = true; - datFile.Add(game, datItem); + datFile.Items.Add(game, datItem); } } } @@ -1618,34 +1612,34 @@ namespace SabreTools.Library.DatFiles // Now that every device is accounted for, add the new list of devices, if they don't already exist foreach (string device in newdevs) { - if (!datFile[game][0].Devices.Contains(device)) - datFile[game][0].Devices.Add(device); + if (!datFile.Items[game][0].Devices.Contains(device)) + datFile.Items[game][0].Devices.Add(device); } // If we're checking slotoptions too if (slotoptions) { // Determine if the game has any slotoptions or not - List slotopts = datFile[game][0].SlotOptions; + List slotopts = datFile.Items[game][0].SlotOptions; List newslotopts = new List(); foreach (string slotopt in slotopts) { // If the slotoption doesn't exist then we continue - if (datFile[slotopt].Count == 0) + if (datFile.Items[slotopt].Count == 0) continue; // Otherwise, copy the items from the slotoption to the current game - DatItem copyFrom = datFile[game][0]; - List slotItems = datFile[slotopt]; + DatItem copyFrom = datFile.Items[game][0]; + List slotItems = datFile.Items[slotopt]; foreach (DatItem item in slotItems) { DatItem datItem = (DatItem)item.Clone(); newslotopts.AddRange(datItem.SlotOptions ?? new List()); datItem.CopyMachineInformation(copyFrom); - if (datFile[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) + if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) { foundnew = true; - datFile.Add(game, datItem); + datFile.Items.Add(game, datItem); } } } @@ -1653,8 +1647,8 @@ namespace SabreTools.Library.DatFiles // Now that every slotoption is accounted for, add the new list of slotoptions, if they don't already exist foreach (string slotopt in newslotopts) { - if (!datFile[game][0].SlotOptions.Contains(slotopt)) - datFile[game][0].SlotOptions.Add(slotopt); + if (!datFile.Items[game][0].SlotOptions.Contains(slotopt)) + datFile.Items[game][0].SlotOptions.Add(slotopt); } } } @@ -1668,45 +1662,43 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void AddRomsFromParent(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue - if (datFile[game].Count == 0) + if (datFile.Items[game].Count == 0) continue; // Determine if the game has a parent or not string parent = null; - if (!string.IsNullOrWhiteSpace(datFile[game][0].CloneOf)) - parent = datFile[game][0].CloneOf; + if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) + parent = datFile.Items[game][0].CloneOf; // If the parent doesnt exist, we want to continue if (string.IsNullOrWhiteSpace(parent)) continue; // If the parent doesn't have any items, we want to continue - if (datFile[parent].Count == 0) + if (datFile.Items[parent].Count == 0) continue; // If the parent exists and has items, we copy the items from the parent to the current game - DatItem copyFrom = datFile[game][0]; - List parentItems = datFile[parent]; + DatItem copyFrom = datFile.Items[game][0]; + List parentItems = datFile.Items[parent]; foreach (DatItem item in parentItems) { DatItem datItem = (DatItem)item.Clone(); datItem.CopyMachineInformation(copyFrom); - if (datFile[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0 - && !datFile[game].Contains(datItem)) + if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0 + && !datFile.Items[game].Contains(datItem)) { - datFile.Add(game, datItem); + datFile.Items.Add(game, datItem); } } // Now we want to get the parent romof tag and put it in each of the items - List items = datFile[game]; - string romof = datFile[parent][0].RomOf; + List items = datFile.Items[game]; + string romof = datFile.Items[parent][0].RomOf; foreach (DatItem item in items) { item.RomOf = romof; @@ -1720,27 +1712,25 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void AddRomsFromChildren(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue - if (datFile[game].Count == 0) + if (datFile.Items[game].Count == 0) continue; // Determine if the game has a parent or not string parent = null; - if (!string.IsNullOrWhiteSpace(datFile[game][0].CloneOf)) - parent = datFile[game][0].CloneOf; + if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) + parent = datFile.Items[game][0].CloneOf; // If there is no parent, then we continue if (string.IsNullOrWhiteSpace(parent)) continue; // Otherwise, move the items from the current game to a subfolder of the parent game - DatItem copyFrom = datFile[parent].Count == 0 ? new Rom { MachineName = parent, MachineDescription = parent } : datFile[parent][0]; - List items = datFile[game]; + DatItem copyFrom = datFile.Items[parent].Count == 0 ? new Rom { MachineName = parent, MachineDescription = parent } : datFile.Items[parent][0]; + List items = datFile.Items[game]; foreach (DatItem item in items) { // Special disk handling @@ -1749,23 +1739,23 @@ namespace SabreTools.Library.DatFiles Disk disk = item as Disk; // If the merge tag exists and the parent already contains it, skip - if (disk.MergeTag != null && datFile[parent].Select(i => i.Name).Contains(disk.MergeTag)) + if (disk.MergeTag != null && datFile.Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) { continue; } // If the merge tag exists but the parent doesn't contain it, add to parent - else if (disk.MergeTag != null && !datFile[parent].Select(i => i.Name).Contains(disk.MergeTag)) + else if (disk.MergeTag != null && !datFile.Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) { item.CopyMachineInformation(copyFrom); - datFile.Add(parent, item); + datFile.Items.Add(parent, item); } // If there is no merge tag, add to parent else if (disk.MergeTag == null) { item.CopyMachineInformation(copyFrom); - datFile.Add(parent, item); + datFile.Items.Add(parent, item); } } @@ -1775,39 +1765,39 @@ namespace SabreTools.Library.DatFiles Rom rom = item as Rom; // If the merge tag exists and the parent already contains it, skip - if (rom.MergeTag != null && datFile[parent].Select(i => i.Name).Contains(rom.MergeTag)) + if (rom.MergeTag != null && datFile.Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) { continue; } // If the merge tag exists but the parent doesn't contain it, add to subfolder of parent - else if (rom.MergeTag != null && !datFile[parent].Select(i => i.Name).Contains(rom.MergeTag)) + else if (rom.MergeTag != null && !datFile.Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) { item.Name = $"{item.MachineName}\\{item.Name}"; item.CopyMachineInformation(copyFrom); - datFile.Add(parent, item); + datFile.Items.Add(parent, item); } // If the parent doesn't already contain this item, add to subfolder of parent - else if (!datFile[parent].Contains(item)) + else if (!datFile.Items[parent].Contains(item)) { item.Name = $"{item.MachineName}\\{item.Name}"; item.CopyMachineInformation(copyFrom); - datFile.Add(parent, item); + datFile.Items.Add(parent, item); } } // All other that would be missing to subfolder of parent - else if (!datFile[parent].Contains(item)) + else if (!datFile.Items[parent].Contains(item)) { item.Name = $"{item.MachineName}\\{item.Name}"; item.CopyMachineInformation(copyFrom); - datFile.Add(parent, item); + datFile.Items.Add(parent, item); } } // Then, remove the old game so it's not picked up by the writer - datFile.Remove(game); + datFile.Items.Remove(game); } } @@ -1817,16 +1807,14 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void RemoveBiosAndDeviceSets(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { - if (datFile[game].Count > 0 - && (datFile[game][0].MachineType.HasFlag(MachineType.Bios) - || datFile[game][0].MachineType.HasFlag(MachineType.Device))) + if (datFile.Items[game].Count > 0 + && (datFile.Items[game][0].MachineType.HasFlag(MachineType.Bios) + || datFile.Items[game][0].MachineType.HasFlag(MachineType.Device))) { - datFile.Remove(game); + datFile.Items.Remove(game); } } } @@ -1839,40 +1827,38 @@ namespace SabreTools.Library.DatFiles private void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false) { // Loop through the romof tags - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue - if (datFile[game].Count == 0) + if (datFile.Items[game].Count == 0) continue; // If the game (is/is not) a bios, we want to continue - if (bios ^ datFile[game][0].MachineType.HasFlag(MachineType.Bios)) + if (bios ^ datFile.Items[game][0].MachineType.HasFlag(MachineType.Bios)) continue; // Determine if the game has a parent or not string parent = null; - if (!string.IsNullOrWhiteSpace(datFile[game][0].RomOf)) - parent = datFile[game][0].RomOf; + if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].RomOf)) + parent = datFile.Items[game][0].RomOf; // If the parent doesnt exist, we want to continue if (string.IsNullOrWhiteSpace(parent)) continue; // If the parent doesn't have any items, we want to continue - if (datFile[parent].Count == 0) + if (datFile.Items[parent].Count == 0) continue; // If the parent exists and has items, we remove the items that are in the parent from the current game - List parentItems = datFile[parent]; + List parentItems = datFile.Items[parent]; foreach (DatItem item in parentItems) { DatItem datItem = (DatItem)item.Clone(); - while (datFile[game].Contains(datItem)) + while (datFile.Items[game].Contains(datItem)) { - datFile.Remove(game, datItem); + datFile.Items.Remove(game, datItem); } } } @@ -1884,42 +1870,40 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void RemoveRomsFromChild(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue - if (datFile[game].Count == 0) + if (datFile.Items[game].Count == 0) continue; // Determine if the game has a parent or not string parent = null; - if (!string.IsNullOrWhiteSpace(datFile[game][0].CloneOf)) - parent = datFile[game][0].CloneOf; + if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) + parent = datFile.Items[game][0].CloneOf; // If the parent doesnt exist, we want to continue if (string.IsNullOrWhiteSpace(parent)) continue; // If the parent doesn't have any items, we want to continue - if (datFile[parent].Count == 0) + if (datFile.Items[parent].Count == 0) continue; // If the parent exists and has items, we remove the parent items from the current game - List parentItems = datFile[parent]; + List parentItems = datFile.Items[parent]; foreach (DatItem item in parentItems) { DatItem datItem = (DatItem)item.Clone(); - while (datFile[game].Contains(datItem)) + while (datFile.Items[game].Contains(datItem)) { - datFile.Remove(game, datItem); + datFile.Items.Remove(game, datItem); } } // Now we want to get the parent romof tag and put it in each of the remaining items - List items = datFile[game]; - string romof = datFile[parent][0].RomOf; + List items = datFile.Items[game]; + string romof = datFile.Items[parent][0].RomOf; foreach (DatItem item in items) { item.RomOf = romof; @@ -1933,12 +1917,10 @@ namespace SabreTools.Library.DatFiles /// DatFile to filter private void RemoveTagsFromChild(DatFile datFile) { - List games = datFile.Keys; - games = games.OrderBy(g => g, NaturalComparer.Default).ToList(); - + List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) { - List items = datFile[game]; + List items = datFile.Items[game]; foreach (DatItem item in items) { item.CloneOf = null; @@ -1962,10 +1944,9 @@ namespace SabreTools.Library.DatFiles { // First we want to get a mapping for all games to description ConcurrentDictionary mapping = new ConcurrentDictionary(); - List keys = datFile.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = datFile[key]; + List items = datFile.Items[key]; foreach (DatItem item in items) { // If the key mapping doesn't exist, add it @@ -1974,10 +1955,9 @@ namespace SabreTools.Library.DatFiles }); // Now we loop through every item and update accordingly - keys = datFile.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = datFile[key]; + List items = datFile.Items[key]; List newItems = new List(); foreach (DatItem item in items) { @@ -2002,8 +1982,8 @@ namespace SabreTools.Library.DatFiles } // Replace the old list of roms with the new one - datFile.Remove(key); - datFile.AddRange(key, newItems); + datFile.Items.Remove(key); + datFile.Items.AddRange(key, newItems); }); } catch (Exception ex) @@ -2020,9 +2000,9 @@ namespace SabreTools.Library.DatFiles private void OneRomPerGame(DatFile datFile) { // For each rom, we want to update the game to be "/" - Parallel.ForEach(datFile.Keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = datFile[key]; + List items = datFile.Items[key]; for (int i = 0; i < items.Count; i++) { string[] splitname = items[i].Name.Split('.'); @@ -2044,10 +2024,9 @@ namespace SabreTools.Library.DatFiles string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)"; // Now process all of the roms - List keys = datFile.Keys; - Parallel.ForEach(keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = datFile[key]; + List items = datFile.Items[key]; for (int j = 0; j < items.Count; j++) { DatItem item = items[j]; @@ -2060,8 +2039,8 @@ namespace SabreTools.Library.DatFiles items[j] = item; } - datFile.Remove(key); - datFile.AddRange(key, items); + datFile.Items.Remove(key); + datFile.Items.AddRange(key, items); }); } diff --git a/SabreTools.Library/DatFiles/Hashfile.cs b/SabreTools.Library/DatFiles/Hashfile.cs index b8f3c4cc..01402e16 100644 --- a/SabreTools.Library/DatFiles/Hashfile.cs +++ b/SabreTools.Library/DatFiles/Hashfile.cs @@ -126,9 +126,9 @@ namespace SabreTools.Library.DatFiles }; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/ItemDictionary.cs b/SabreTools.Library/DatFiles/ItemDictionary.cs new file mode 100644 index 00000000..13d0c2f2 --- /dev/null +++ b/SabreTools.Library/DatFiles/ItemDictionary.cs @@ -0,0 +1,576 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using SabreTools.Library.Data; +using SabreTools.Library.DatItems; +using NaturalSort; + +namespace SabreTools.Library.DatFiles +{ + /// + /// Item dictionary with statistics, bucketing, and sorting + /// + public class ItemDictionary : IDictionary> + { + #region Private instance variables + + /// + /// Determine the bucketing key for all items + /// + private BucketedBy bucketedBy; + + /// + /// Determine merging type for all items + /// + private DedupeType mergedBy; + + /// + /// Internal dictionary for the class + /// + private Dictionary> items; + + #endregion + + #region Publically available fields + + /// + /// DatStats object for reporting + /// + public DatStats Statistics { get; set; } + + /// + /// Get the keys from the file dictionary + /// + /// List of the keys + public ICollection Keys + { + get { return items.Keys; } + } + + /// + /// Get the keys in sorted order from the file dictionary + /// + /// List of the keys in sorted order + public List SortedKeys + { + get + { + var keys = items.Keys.ToList(); + keys.Sort(new NaturalComparer()); + return keys; + } + } + + #endregion + + #region Accessors + + /// + /// Passthrough to access the file dictionary + /// + /// Key in the dictionary to reference + public List this[string key] + { + get + { + // Explicit lock for some weird corner cases + lock (key) + { + // Ensure the key exists + EnsureKey(key); + + // Now return the value + return items[key]; + } + } + set + { + AddRange(key, value); + } + } + + /// + /// Add a value to the file dictionary + /// + /// Key in the dictionary to add to + /// Value to add to the dictionary + public void Add(string key, DatItem value) + { + // Explicit lock for some weird corner cases + lock (key) + { + // Ensure the key exists + EnsureKey(key); + + // If item is null, don't add it + if (value == null) + return; + + // Now add the value + items[key].Add(value); + + // Now update the statistics + Statistics.AddItem(value); + } + } + + /// + /// Add a range of values to the file dictionary + /// + /// Key in the dictionary to add to + /// Value to add to the dictionary + public void Add(string key, List value) + { + AddRange(key, value); + } + + /// + /// Add a range of values to the file dictionary + /// + /// Key in the dictionary to add to + /// Value to add to the dictionary + public void AddRange(string key, List value) + { + // Explicit lock for some weird corner cases + lock (key) + { + // Ensure the key exists + EnsureKey(key); + + // Now add the value + items[key].AddRange(value); + + // Now update the statistics + foreach (DatItem item in value) + { + Statistics.AddItem(item); + } + } + } + + /// + /// Get if the file dictionary contains the key + /// + /// Key in the dictionary to check + /// True if the key exists, false otherwise + public bool ContainsKey(string key) + { + // If the key is null, we return false since keys can't be null + if (key == null) + return false; + + // Explicit lock for some weird corner cases + lock (key) + { + return items.ContainsKey(key); + } + } + + /// + /// Get if the file dictionary contains the key and value + /// + /// Key in the dictionary to check + /// Value in the dictionary to check + /// True if the key exists, false otherwise + public bool Contains(string key, DatItem value) + { + // If the key is null, we return false since keys can't be null + if (key == null) + return false; + + // Explicit lock for some weird corner cases + lock (key) + { + if (items.ContainsKey(key)) + return items[key].Contains(value); + } + + return false; + } + + /// + /// Remove a key from the file dictionary if it exists + /// + /// Key in the dictionary to remove + public bool Remove(string key) + { + // If the key doesn't exist, return + if (!ContainsKey(key)) + return false; + + // Remove the statistics first + foreach (DatItem item in items[key]) + { + Statistics.RemoveItem(item); + } + + // Remove the key from the dictionary + return items.Remove(key); + } + + /// + /// Remove the first instance of a value from the file dictionary if it exists + /// + /// Key in the dictionary to remove from + /// Value to remove from the dictionary + public bool Remove(string key, DatItem value) + { + // If the key and value doesn't exist, return + if (!Contains(key, value)) + return false; + + // Remove the statistics first + Statistics.RemoveItem(value); + + return items[key].Remove(value); + } + + /// + /// Override the internal BucketedBy value + /// + /// + public void SetBucketedBy(BucketedBy newBucket) + { + bucketedBy = newBucket; + } + + /// + /// Ensure the key exists in the items dictionary + /// + /// Key to ensure + private void EnsureKey(string key) + { + // If the key is missing from the dictionary, add it + if (!items.ContainsKey(key)) + items.Add(key, new List()); + } + + #endregion + + #region Constructors + + /// + /// Generic constructor + /// + public ItemDictionary() + { + bucketedBy = BucketedBy.Default; + mergedBy = DedupeType.None; + items = new Dictionary>(); + + Statistics = new DatStats(); + } + + /// + /// Constructor for statistics only + /// + /// Existing statistics to pre-populate + public ItemDictionary(DatStats stats) + { + bucketedBy = BucketedBy.Default; + mergedBy = DedupeType.None; + items = new Dictionary>(); + + Statistics = stats; + } + + #endregion + + #region Custom Functionality + + /// + /// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method + /// + /// BucketedBy enum representing how to bucket the individual items + /// Dedupe type that should be used + /// True if the key should be lowercased (default), false otherwise + /// True if games should only be compared on game and file name, false if system and source are counted + public void BucketBy(BucketedBy bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true) + { + // If we have a situation where there's no dictionary or no keys at all, we skip + if (items == null || items.Count == 0) + return; + + // If the sorted type isn't the same, we want to sort the dictionary accordingly + if (this.bucketedBy != bucketBy) + { + Globals.Logger.User($"Organizing roms by {bucketBy}"); + + // Set the sorted type + this.bucketedBy = bucketBy; + + // Reset the merged type since this might change the merge + this.mergedBy = DedupeType.None; + + // First do the initial sort of all of the roms inplace + List oldkeys = Keys.ToList(); + for (int k = 0; k < oldkeys.Count; k++) + { + string key = oldkeys[k]; + + // Get the unsorted current list + List items = this[key]; + + // Now add each of the roms to their respective keys + for (int i = 0; i < items.Count; i++) + { + DatItem item = items[i]; + if (item == null) + continue; + + // We want to get the key most appropriate for the given sorting type + string newkey = item.GetKey(bucketBy, lower, norename); + + // If the key is different, move the item to the new key + if (newkey != key) + { + Add(newkey, item); + Remove(key, item); + i--; // This make sure that the pointer stays on the correct since one was removed + } + } + + // If the key is now empty, remove it + if (this[key].Count == 0) + Remove(key); + } + } + + // If the merge type isn't the same, we want to merge the dictionary accordingly + if (this.mergedBy != dedupeType) + { + Globals.Logger.User($"Deduping roms by {dedupeType}"); + + // Set the sorted type + this.mergedBy = dedupeType; + + Parallel.ForEach(Keys, Globals.ParallelOptions, key => + { + // Get the possibly unsorted list + List sortedlist = this[key]; + + // Sort the list of items to be consistent + DatItem.Sort(ref sortedlist, false); + + // If we're merging the roms, do so + if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == BucketedBy.Game)) + sortedlist = DatItem.Merge(sortedlist); + + // Add the list back to the dictionary + Remove(key); + AddRange(key, sortedlist); + }); + } + // If the merge type is the same, we want to sort the dictionary to be consistent + else + { + Parallel.ForEach(Keys, Globals.ParallelOptions, key => + { + // Get the possibly unsorted list + List sortedlist = this[key]; + + // Sort the list of items to be consistent + DatItem.Sort(ref sortedlist, false); + }); + } + + // Now clean up all empty keys + ClearEmpty(); + } + + /// + /// Remove all items marked for removal + /// + public void ClearMarked() + { + foreach (string key in items.Keys) + { + List oldItemList = items[key]; + List newItemList = oldItemList.Where(i => !i.Remove).ToList(); + + Remove(key); + AddRange(key, newItemList); + } + } + + /// + /// List all duplicates found in a DAT based on a DatItem + /// + /// Item to try to match + /// True to mark matched roms for removal from the input, false otherwise (default) + /// True if the DAT is already sorted accordingly, false otherwise (default) + /// List of matched DatItem objects + public List GetDuplicates(DatItem datItem, bool remove = false, bool sorted = false) + { + List output = new List(); + + // Check for an empty rom list first + if (Statistics.Count == 0) + return output; + + // We want to get the proper key for the DatItem + string key = SortAndGetKey(datItem, sorted); + + // If the key doesn't exist, return the empty list + if (!ContainsKey(key)) + return output; + + // Try to find duplicates + List roms = this[key]; + List left = new List(); + for (int i = 0; i < roms.Count; i++) + { + DatItem other = roms[i]; + + if (datItem.Equals(other)) + { + other.Remove = true; + output.Add(other); + } + else + { + left.Add(other); + } + } + + // If we're in removal mode, add back all roms with the proper flags + if (remove) + { + Remove(key); + AddRange(key, output); + AddRange(key, left); + } + + return output; + } + + /// + /// Check if a DAT contains the given DatItem + /// + /// Item to try to match + /// True if the DAT is already sorted accordingly, false otherwise (default) + /// True if it contains the rom, false otherwise + public bool HasDuplicates(DatItem datItem, bool sorted = false) + { + // Check for an empty rom list first + if (Statistics.Count == 0) + return false; + + // We want to get the proper key for the DatItem + string key = SortAndGetKey(datItem, sorted); + + // If the key doesn't exist, return the empty list + if (!ContainsKey(key)) + return false; + + // Try to find duplicates + List roms = this[key]; + return roms.Any(r => datItem.Equals(r)); + } + + /// + /// Recalculate the statistics for the Dat + /// + public void RecalculateStats() + { + // Wipe out any stats already there + Statistics.Reset(); + + // If we have a blank Dat in any way, return + if (this == null || Statistics.Count == 0) + return; + + // Loop through and add + Parallel.ForEach(items.Keys, Globals.ParallelOptions, key => + { + List datItems = items[key]; + foreach (DatItem item in datItems) + { + Statistics.AddItem(item); + } + }); + } + + /// + /// Remove any keys that have null or empty values + /// + private void ClearEmpty() + { + foreach (string key in items.Keys) + { + if (items[key] == null || items[key].Count == 0) + items.Remove(key); + } + } + + /// + /// Sort the input DAT and get the key to be used by the item + /// + /// Item to try to match + /// True if the DAT is already sorted accordingly, false otherwise (default) + /// Key to try to use + private string SortAndGetKey(DatItem datItem, bool sorted = false) + { + // If we're not already sorted, take care of it + if (!sorted) + BucketBy(Statistics.GetBestAvailable(), DedupeType.None); + + // Now that we have the sorted type, we get the proper key + return datItem.GetKey(bucketedBy); + } + + #endregion + + #region IDictionary Implementations + + public ICollection> Values => ((IDictionary>)items).Values; + + public int Count => ((ICollection>>)items).Count; + + public bool IsReadOnly => ((ICollection>>)items).IsReadOnly; + + public bool TryGetValue(string key, out List value) + { + return ((IDictionary>)items).TryGetValue(key, out value); + } + + public void Add(KeyValuePair> item) + { + ((ICollection>>)items).Add(item); + } + + public void Clear() + { + ((ICollection>>)items).Clear(); + } + + public bool Contains(KeyValuePair> item) + { + return ((ICollection>>)items).Contains(item); + } + + public void CopyTo(KeyValuePair>[] array, int arrayIndex) + { + ((ICollection>>)items).CopyTo(array, arrayIndex); + } + + public bool Remove(KeyValuePair> item) + { + return ((ICollection>>)items).Remove(item); + } + + public IEnumerator>> GetEnumerator() + { + return ((IEnumerable>>)items).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + #endregion + } +} diff --git a/SabreTools.Library/DatFiles/Json.cs b/SabreTools.Library/DatFiles/Json.cs index d27ea57d..ee111021 100644 --- a/SabreTools.Library/DatFiles/Json.cs +++ b/SabreTools.Library/DatFiles/Json.cs @@ -777,9 +777,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/Listrom.cs b/SabreTools.Library/DatFiles/Listrom.cs index 5f2a841a..3fce4ea1 100644 --- a/SabreTools.Library/DatFiles/Listrom.cs +++ b/SabreTools.Library/DatFiles/Listrom.cs @@ -254,9 +254,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/Listxml.cs b/SabreTools.Library/DatFiles/Listxml.cs index 77fc4061..0ffe12f8 100644 --- a/SabreTools.Library/DatFiles/Listxml.cs +++ b/SabreTools.Library/DatFiles/Listxml.cs @@ -575,9 +575,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/Logiqx.cs b/SabreTools.Library/DatFiles/Logiqx.cs index 253fe7ea..9830abc4 100644 --- a/SabreTools.Library/DatFiles/Logiqx.cs +++ b/SabreTools.Library/DatFiles/Logiqx.cs @@ -672,9 +672,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/Missfile.cs b/SabreTools.Library/DatFiles/Missfile.cs index 9f713d41..ed5d0405 100644 --- a/SabreTools.Library/DatFiles/Missfile.cs +++ b/SabreTools.Library/DatFiles/Missfile.cs @@ -68,9 +68,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/OfflineList.cs b/SabreTools.Library/DatFiles/OfflineList.cs index da94c707..72a87b95 100644 --- a/SabreTools.Library/DatFiles/OfflineList.cs +++ b/SabreTools.Library/DatFiles/OfflineList.cs @@ -758,9 +758,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); @@ -837,7 +837,7 @@ namespace SabreTools.Library.DatFiles xtw.WriteStartElement("configuration"); xtw.WriteElementString("datName", DatHeader.Name); - xtw.WriteElementString("datVersion", DatStats.Count.ToString()); + xtw.WriteElementString("datVersion", Items.Statistics.Count.ToString()); xtw.WriteElementString("system", "none"); xtw.WriteElementString("screenshotsWidth", "240"); xtw.WriteElementString("screenshotsHeight", "160"); diff --git a/SabreTools.Library/DatFiles/OpenMSX.cs b/SabreTools.Library/DatFiles/OpenMSX.cs index dee9a90a..d25814e7 100644 --- a/SabreTools.Library/DatFiles/OpenMSX.cs +++ b/SabreTools.Library/DatFiles/OpenMSX.cs @@ -502,9 +502,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/RomCenter.cs b/SabreTools.Library/DatFiles/RomCenter.cs index e9e16093..e48f044c 100644 --- a/SabreTools.Library/DatFiles/RomCenter.cs +++ b/SabreTools.Library/DatFiles/RomCenter.cs @@ -393,9 +393,9 @@ namespace SabreTools.Library.DatFiles List splitpath = new List(); // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/SabreDat.cs b/SabreTools.Library/DatFiles/SabreDat.cs index cf5c9803..54906f39 100644 --- a/SabreTools.Library/DatFiles/SabreDat.cs +++ b/SabreTools.Library/DatFiles/SabreDat.cs @@ -564,9 +564,9 @@ namespace SabreTools.Library.DatFiles List splitpath = new List(); // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/SeparatedValue.cs b/SabreTools.Library/DatFiles/SeparatedValue.cs index b9ab50ce..486961d7 100644 --- a/SabreTools.Library/DatFiles/SeparatedValue.cs +++ b/SabreTools.Library/DatFiles/SeparatedValue.cs @@ -883,9 +883,9 @@ namespace SabreTools.Library.DatFiles WriteHeader(svw); // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); diff --git a/SabreTools.Library/DatFiles/SoftwareList.cs b/SabreTools.Library/DatFiles/SoftwareList.cs index 7c9f8a86..72833921 100644 --- a/SabreTools.Library/DatFiles/SoftwareList.cs +++ b/SabreTools.Library/DatFiles/SoftwareList.cs @@ -375,14 +375,14 @@ namespace SabreTools.Library.DatFiles // If the rom is continue or ignore, add the size to the previous rom if (reader.GetAttribute("loadflag") == "continue" || reader.GetAttribute("loadflag") == "ignore") { - int index = this[key].Count - 1; - DatItem lastrom = this[key][index]; + int index = Items[key].Count - 1; + DatItem lastrom = Items[key][index]; if (lastrom.ItemType == ItemType.Rom) { ((Rom)lastrom).Size += Sanitizer.CleanSize(reader.GetAttribute("size")); } - this[key].RemoveAt(index); - this[key].Add(lastrom); + Items[key].RemoveAt(index); + Items[key].Add(lastrom); reader.Read(); continue; } @@ -555,9 +555,9 @@ namespace SabreTools.Library.DatFiles string lastgame = null; // Use a sorted list of games to output - foreach (string key in SortedKeys) + foreach (string key in Items.SortedKeys) { - List roms = this[key]; + List roms = Items[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms);