diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs b/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs index 4b8a7677..00d4b65e 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs @@ -35,96 +35,92 @@ namespace SabreTools.Helper.Dats // Create the temporary dictionary to sort into SortedDictionary> sortable = new SortedDictionary>(); - Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms by " + bucketBy); + Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : "")); // First do the initial sort of all of the roms List keys = Keys.ToList(); Parallel.ForEach(keys, new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism }, key => - { - List roms = this[key]; + { + List roms = this[key]; - // If we're merging the roms, do so - if (mergeroms) + // Now add each of the roms to their respective games + Parallel.ForEach(roms, + new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism }, + rom => + { + string newkey = ""; + + // We want to get the key most appropriate for the given sorting type + switch (bucketBy) { - roms = DatItem.Merge(roms); + case SortedBy.CRC: + newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); + break; + case SortedBy.Game: + newkey = (norename ? "" + : rom.SystemID.ToString().PadLeft(10, '0') + + "-" + + rom.SourceID.ToString().PadLeft(10, '0') + "-") + + (String.IsNullOrEmpty(rom.Machine.Name) + ? "Default" + : rom.Machine.Name); + if (lower) + { + newkey = newkey.ToLowerInvariant(); + } + + newkey = HttpUtility.HtmlEncode(newkey); + break; + case SortedBy.MD5: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).MD5 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).MD5 + : Constants.MD5Zero)); + break; + case SortedBy.SHA1: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA1 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA1 + : Constants.SHA1Zero)); + break; + case SortedBy.SHA256: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA256 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA256 + : Constants.SHA256Zero)); + break; + case SortedBy.SHA384: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA384 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA384 + : Constants.SHA384Zero)); + break; + case SortedBy.SHA512: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA512 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA512 + : Constants.SHA512Zero)); + break; } - // Now add each of the roms to their respective games - foreach (DatItem rom in roms) + // Add the DatItem to the temp dictionary + lock (sortable) { - string newkey = ""; - - // We want to get the key most appropriate for the given sorting type - switch (bucketBy) + if (!sortable.ContainsKey(newkey)) { - case SortedBy.CRC: - newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); - break; - case SortedBy.Game: - newkey = (norename ? "" - : rom.SystemID.ToString().PadLeft(10, '0') - + "-" - + rom.SourceID.ToString().PadLeft(10, '0') + "-") - + (String.IsNullOrEmpty(rom.Machine.Name) - ? "Default" - : rom.Machine.Name); - if (lower) - { - newkey = newkey.ToLowerInvariant(); - } - - newkey = HttpUtility.HtmlEncode(newkey); - break; - case SortedBy.MD5: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).MD5 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).MD5 - : Constants.MD5Zero)); - break; - case SortedBy.SHA1: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA1 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA1 - : Constants.SHA1Zero)); - break; - case SortedBy.SHA256: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA256 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA256 - : Constants.SHA256Zero)); - break; - case SortedBy.SHA384: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA384 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA384 - : Constants.SHA384Zero)); - break; - case SortedBy.SHA512: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA512 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA512 - : Constants.SHA512Zero)); - break; - } - - // Add the DatItem to the temp dictionary - lock (sortable) - { - if (!sortable.ContainsKey(newkey)) - { - sortable.Add(newkey, new List()); - } - sortable[newkey].Add(rom); + sortable.Add(newkey, new List()); } + sortable[newkey].Add(rom); } }); + }); // Now go through and sort all of the individual lists keys = sortable.Keys.ToList(); @@ -132,9 +128,19 @@ namespace SabreTools.Helper.Dats new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism }, key => { + // Get the possibly unsorted list List sortedlist = sortable[key]; + + // Sort the list of items to be consistent DatItem.Sort(ref sortedlist, false); + // If we're merging the roms, do so + if (mergeroms) + { + sortedlist = DatItem.Merge(sortedlist); + } + + // Add the list back to the temp dictionary lock (sortable) { sortable[key] = sortedlist; diff --git a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs index c852f9cb..a8516e68 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs @@ -100,8 +100,7 @@ namespace SabreTools.Helper.Dats DateTime start = DateTime.Now; Globals.Logger.User("Processing individual DATs"); - // TODO: Can parsing headers be separated from parsing content? - // TODO: Can all DATs be parsed into the same structure in one loop? + // Parse all of the DATs into their own DatFiles in the array Parallel.For(0, inputs.Count, new ParallelOptions { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism }, diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs index f1e5a5e5..97558bc0 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs @@ -38,8 +38,7 @@ namespace SabreTools.Helper.Dats /// The following features have been requested for file output: /// - Have the ability to strip special (non-ASCII) characters from rom information /// - public bool WriteToFile(string outDir, - bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) + public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) { // If there's nothing there, abort if (Count == 0) @@ -109,8 +108,14 @@ namespace SabreTools.Helper.Dats recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true); } - // Bucket roms by game name and optionally dedupe - BucketBy(SortedBy.Game, MergeRoms, norename: norename); + // First bucket by CRC to dedupe if required + if (MergeRoms) + { + BucketBy(SortedBy.CRC, MergeRoms, norename: norename); + } + + // Bucket roms by game name + BucketBy(SortedBy.Game, false /* mergeRoms */, norename: norename); // Output the number of items we're going to be writing Globals.Logger.User("A total of " + Count + " items will be written out to file");