diff --git a/SabreTools.Helper/Dats/DatFile.cs b/SabreTools.Helper/Dats/DatFile.cs index 09a20216..acee4da4 100644 --- a/SabreTools.Helper/Dats/DatFile.cs +++ b/SabreTools.Helper/Dats/DatFile.cs @@ -1,5 +1,5 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; +using System.Threading.Tasks; using SabreTools.Helper.Data; @@ -460,10 +460,10 @@ namespace SabreTools.Helper.Dats lock (_files) { int count = 0; - foreach (string key in _files.Keys) + Parallel.ForEach(_files.Keys, Globals.ParallelOptions, key => { count += _files[key].Count; - } + }); return count; } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs index 9ed11a6a..5d37b860 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs @@ -221,51 +221,57 @@ namespace SabreTools.Helper.Dats // Now, loop through the dictionary and populate the correct DATs start = DateTime.Now; Globals.Logger.User("Populating all output DATs"); + List keys = Keys.ToList(); - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - List roms = DatItem.Merge(this[key]); + List items = DatItem.Merge(this[key]); - if (roms != null && roms.Count > 0) + // If the rom list is empty or null, just skip it + if (items == null || items.Count == 0) { - foreach (DatItem rom in roms) + return; + } + + // Loop through and add the items correctly + Parallel.ForEach(items, Globals.ParallelOptions, item => + { + // No duplicates + if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0) { - // No duplicates - if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0) + if ((item.Dupe & DupeType.Internal) != 0) { - if ((rom.Dupe & DupeType.Internal) != 0) + // Individual DATs that are output + if ((diff & DiffMode.Individuals) != 0) { - // Individual DATs that are output - if ((diff & DiffMode.Individuals) != 0) - { - outDats[rom.SystemID].Add(key, rom); - } - - // Merged no-duplicates DAT - if ((diff & DiffMode.NoDupes) != 0) - { - DatItem newrom = rom; - newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")"; - - outerDiffData.Add(key, newrom); - } + outDats[item.SystemID].Add(key, item); } - } - // Duplicates only - if ((diff & DiffMode.Dupes) != 0) - { - if ((rom.Dupe & DupeType.External) != 0) + // Merged no-duplicates DAT + if ((diff & DiffMode.NoDupes) != 0) { - DatItem newrom = rom; + DatItem newrom = item; newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")"; - dupeData.Add(key, newrom); + outerDiffData.Add(key, newrom); } } } - } - } + + // Duplicates only + if ((diff & DiffMode.Dupes) != 0) + { + if ((item.Dupe & DupeType.External) != 0) + { + DatItem newrom = item; + newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")"; + + dupeData.Add(key, newrom); + } + } + }); + }); + Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); // Finally, loop through and output each of the DATs @@ -287,7 +293,7 @@ namespace SabreTools.Helper.Dats // Output the individual (a-b) DATs if ((diff & DiffMode.Individuals) != 0) { - for (int j = 0; j < inputs.Count; j++) + Parallel.For(0, inputs.Count, j => { // If we have an output directory set, replace the path string[] split = inputs[j].Split('¬'); @@ -297,7 +303,7 @@ namespace SabreTools.Helper.Dats // Try to output the file outDats[j].WriteToFile(path); - } + }); } Globals.Logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); } @@ -353,31 +359,36 @@ namespace SabreTools.Helper.Dats Globals.Logger.User("Populating all output DATs"); List keys = Keys.ToList(); - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - List roms = DatItem.Merge(this[key]); + List items = DatItem.Merge(this[key]); - if (roms != null && roms.Count > 0) + // If the rom list is empty or null, just skip it + if (items == null || items.Count == 0) { - foreach (DatItem rom in roms) - { - // There's odd cases where there are items with System ID < 0. Skip them for now - if (rom.SystemID < 0) - { - Globals.Logger.Warning("Item found with a <0 SystemID: " + rom.Name); - continue; - } - - outDats[rom.SystemID].Add(key, rom); - } + return; } - } + + Parallel.ForEach(items, Globals.ParallelOptions, item => + { + // There's odd cases where there are items with System ID < 0. Skip them for now + if (item.SystemID < 0) + { + Globals.Logger.Warning("Item found with a <0 SystemID: " + item.Name); + return; + } + + outDats[item.SystemID].Add(key, item); + }); + }); + Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); // Finally, loop through and output each of the DATs start = DateTime.Now; Globals.Logger.User("Outputting all created DATs"); - for (int j = (skip ? 1 : 0); j < inputs.Count; j++) + + Parallel.For((skip ? 1 : 0), inputs.Count, j => { // If we have an output directory set, replace the path string path = ""; @@ -395,7 +406,8 @@ namespace SabreTools.Helper.Dats // Try to output the file outDats[j].WriteToFile(path); - } + }); + Globals.Logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); } @@ -411,103 +423,37 @@ namespace SabreTools.Helper.Dats if (Type == "SuperDAT") { List keys = Keys.ToList(); - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - List newroms = new List(); - foreach (DatItem rom in this[key]) + List items = this[key].ToList(); + List newItems = new List(); + Parallel.ForEach(items, Globals.ParallelOptions, item => { - DatItem newrom = rom; - string filename = inputs[newrom.SystemID].Split('¬')[0]; - string rootpath = inputs[newrom.SystemID].Split('¬')[1]; + DatItem newItem = item; + string filename = inputs[newItem.SystemID].Split('¬')[0]; + string rootpath = inputs[newItem.SystemID].Split('¬')[1]; rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString()); filename = filename.Remove(0, rootpath.Length); - newrom.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar + newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar - + newrom.Machine.Name; - newroms.Add(newrom); - } - this[key] = newroms; - } + + newItem.Machine.Name; + + lock (newItems) + { + newItems.Add(newItem); + } + }); + + Remove(key); + AddRange(key, newItems); + }); } // Try to output the file WriteToFile(outDir); } - /// - /// Strip the given hash types from the DAT - /// - private void StripHashesFromItems() - { - // Output the logging statement - Globals.Logger.User("Stripping requested hashes"); - - // Now process all of the roms - List keys = Keys.ToList(); - for (int i = 0; i < keys.Count; i++) - { - List items = this[keys[i]]; - for (int j = 0; j < items.Count; j++) - { - DatItem item = items[j]; - if (item.Type == ItemType.Rom) - { - Rom rom = (Rom)item; - if ((StripHash & Hash.MD5) != 0) - { - rom.MD5 = null; - } - if ((StripHash & Hash.SHA1) != 0) - { - rom.SHA1 = null; - } - if ((StripHash & Hash.SHA256) != 0) - { - rom.SHA256 = null; - } - if ((StripHash & Hash.SHA384) != 0) - { - rom.SHA384 = null; - } - if ((StripHash & Hash.SHA512) != 0) - { - rom.SHA512 = null; - } - - items[j] = rom; - } - else if (item.Type == ItemType.Disk) - { - Disk disk = (Disk)item; - if ((StripHash & Hash.MD5) != 0) - { - disk.MD5 = null; - } - if ((StripHash & Hash.SHA1) != 0) - { - disk.SHA1 = null; - } - if ((StripHash & Hash.SHA256) != 0) - { - disk.SHA256 = null; - } - if ((StripHash & Hash.SHA384) != 0) - { - disk.SHA384 = null; - } - if ((StripHash & Hash.SHA512) != 0) - { - disk.SHA512 = null; - } - items[j] = disk; - } - } - - this[keys[i]] = items; - } - } - /// /// Convert, update, and filter a DAT file or set of files using a base /// diff --git a/SabreTools.Helper/Dats/Partials/DatFile.DFD.cs b/SabreTools.Helper/Dats/Partials/DatFile.DFD.cs index 168ec9c0..5bd0a9f4 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.DFD.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.DFD.cs @@ -224,13 +224,13 @@ namespace SabreTools.Helper.Dats else { // First take care of the found items - foreach (Rom rom in extracted) + Parallel.ForEach(extracted, Globals.ParallelOptions, rom => { PopulateFromDirProcessFileHelper(newItem, rom, basePath, (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item)); - } + }); // Then, if we're looking for blanks, get all of the blank folders and add them if (addBlanks) diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs index d527a269..9e3ef362 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs @@ -16,144 +16,144 @@ namespace SabreTools.Helper.Dats { public partial class DatFile { -#region Instance Methods + #region Instance Methods -#region Bucketing + #region Bucketing - /// - /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method - /// - /// SortedBy enum representing how to sort the individual items - /// True if roms should be deduped, false otherwise - /// True if the key should be lowercased (default), false otherwise - /// True if games should only be compared on game and file name, false if system and source are counted - public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true) - { - // If we already have the right sorting, trust it - if (_sortedBy == bucketBy) - { - return; - } - - // Set the sorted type - _sortedBy = bucketBy; - - // Create the temporary dictionary to sort into - SortedDictionary> sortable = new SortedDictionary>(); - - Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : "")); - - // First do the initial sort of all of the roms - List keys = Keys.ToList(); - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - List roms = this[key]; - - // Now add each of the roms to their respective games - Parallel.ForEach(roms, Globals.ParallelOptions, rom => + /// + /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method + /// + /// SortedBy enum representing how to sort the individual items + /// True if roms should be deduped, false otherwise + /// True if the key should be lowercased (default), false otherwise + /// True if games should only be compared on game and file name, false if system and source are counted + public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true) { - string newkey = ""; - - // We want to get the key most appropriate for the given sorting type - switch (bucketBy) + // If we already have the right sorting, trust it + if (_sortedBy == bucketBy) { - case SortedBy.CRC: - newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); - break; - case SortedBy.Game: - newkey = (norename ? "" - : rom.SystemID.ToString().PadLeft(10, '0') - + "-" - + rom.SourceID.ToString().PadLeft(10, '0') + "-") - + (String.IsNullOrEmpty(rom.Machine.Name) - ? "Default" - : rom.Machine.Name); - if (lower) + return; + } + + // Set the sorted type + _sortedBy = bucketBy; + + // Create the temporary dictionary to sort into + SortedDictionary> sortable = new SortedDictionary>(); + + Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : "")); + + // First do the initial sort of all of the roms + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + List roms = this[key]; + + // Now add each of the roms to their respective games + Parallel.ForEach(roms, Globals.ParallelOptions, rom => + { + string newkey = ""; + + // We want to get the key most appropriate for the given sorting type + switch (bucketBy) { - newkey = newkey.ToLowerInvariant(); + case SortedBy.CRC: + newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); + break; + case SortedBy.Game: + newkey = (norename ? "" + : rom.SystemID.ToString().PadLeft(10, '0') + + "-" + + rom.SourceID.ToString().PadLeft(10, '0') + "-") + + (String.IsNullOrEmpty(rom.Machine.Name) + ? "Default" + : rom.Machine.Name); + if (lower) + { + newkey = newkey.ToLowerInvariant(); + } + + newkey = HttpUtility.HtmlEncode(newkey); + break; + case SortedBy.MD5: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).MD5 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).MD5 + : Constants.MD5Zero)); + break; + case SortedBy.SHA1: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA1 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA1 + : Constants.SHA1Zero)); + break; + case SortedBy.SHA256: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA256 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA256 + : Constants.SHA256Zero)); + break; + case SortedBy.SHA384: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA384 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA384 + : Constants.SHA384Zero)); + break; + case SortedBy.SHA512: + newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA512 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA512 + : Constants.SHA512Zero)); + break; } - newkey = HttpUtility.HtmlEncode(newkey); - break; - case SortedBy.MD5: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).MD5 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).MD5 - : Constants.MD5Zero)); - break; - case SortedBy.SHA1: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA1 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA1 - : Constants.SHA1Zero)); - break; - case SortedBy.SHA256: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA256 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA256 - : Constants.SHA256Zero)); - break; - case SortedBy.SHA384: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA384 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA384 - : Constants.SHA384Zero)); - break; - case SortedBy.SHA512: - newkey = (rom.Type == ItemType.Rom - ? ((Rom)rom).SHA512 - : (rom.Type == ItemType.Disk - ? ((Disk)rom).SHA512 - : Constants.SHA512Zero)); - break; - } + // Add the DatItem to the temp dictionary + lock (sortable) + { + if (!sortable.ContainsKey(newkey)) + { + sortable.Add(newkey, new List()); + } + sortable[newkey].Add(rom); + } + }); + }); - // Add the DatItem to the temp dictionary - lock (sortable) + // Now go through and sort all of the individual lists + keys = sortable.Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - if (!sortable.ContainsKey(newkey)) + // Get the possibly unsorted list + List sortedlist = sortable[key]; + + // Sort the list of items to be consistent + DatItem.Sort(ref sortedlist, false); + + // If we're merging the roms, do so + if (mergeroms) { - sortable.Add(newkey, new List()); + sortedlist = DatItem.Merge(sortedlist); } - sortable[newkey].Add(rom); - } - }); - }); - // Now go through and sort all of the individual lists - keys = sortable.Keys.ToList(); - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - // Get the possibly unsorted list - List sortedlist = sortable[key]; + // Add the list back to the temp dictionary + lock (sortable) + { + sortable[key] = sortedlist; + } + }); - // Sort the list of items to be consistent - DatItem.Sort(ref sortedlist, false); - - // If we're merging the roms, do so - if (mergeroms) - { - sortedlist = DatItem.Merge(sortedlist); + // Now assign the dictionary back + _files = sortable; } - // Add the list back to the temp dictionary - lock (sortable) - { - sortable[key] = sortedlist; - } - }); + #endregion - // Now assign the dictionary back - _files = sortable; - } - -#endregion - -#region Filtering + #region Filtering /// /// Filter a DAT based on input parameters and modify the items @@ -284,9 +284,90 @@ namespace SabreTools.Helper.Dats } } -#endregion + /// + /// Strip the given hash types from the DAT + /// + public void StripHashesFromItems() + { + // Output the logging statement + Globals.Logger.User("Stripping requested hashes"); -#region Merging/Splitting Methods + // Now process all of the roms + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + List items = this[key]; + Parallel.For(0, items.Count, Globals.ParallelOptions, j => + { + DatItem item = items[j]; + if (item.Type == ItemType.Rom) + { + Rom rom = (Rom)item; + if ((StripHash & Hash.MD5) != 0) + { + rom.MD5 = null; + } + if ((StripHash & Hash.SHA1) != 0) + { + rom.SHA1 = null; + } + if ((StripHash & Hash.SHA256) != 0) + { + rom.SHA256 = null; + } + if ((StripHash & Hash.SHA384) != 0) + { + rom.SHA384 = null; + } + if ((StripHash & Hash.SHA512) != 0) + { + rom.SHA512 = null; + } + + lock (items) + { + items[j] = rom; + } + } + else if (item.Type == ItemType.Disk) + { + Disk disk = (Disk)item; + if ((StripHash & Hash.MD5) != 0) + { + disk.MD5 = null; + } + if ((StripHash & Hash.SHA1) != 0) + { + disk.SHA1 = null; + } + if ((StripHash & Hash.SHA256) != 0) + { + disk.SHA256 = null; + } + if ((StripHash & Hash.SHA384) != 0) + { + disk.SHA384 = null; + } + if ((StripHash & Hash.SHA512) != 0) + { + disk.SHA512 = null; + } + + lock (items) + { + items[j] = disk; + } + } + }); + + Remove(key); + AddRange(key, items); + }); + } + + #endregion + + #region Merging/Splitting Methods /// /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets @@ -382,7 +463,7 @@ namespace SabreTools.Helper.Dats #endregion -#region Merging/Splitting Helper Methods + #region Merging/Splitting Helper Methods /// /// Use romof tags to add roms to the children @@ -965,6 +1046,6 @@ namespace SabreTools.Helper.Dats #endregion -#endregion // Instance Methods + #endregion // Instance Methods } } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs index e0e2fb50..1a33ff70 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using SabreTools.Helper.Data; using SabreTools.Helper.Skippers; @@ -126,15 +127,18 @@ namespace SabreTools.Helper.Dats // Now loop through and get only directories from the input paths List directories = new List(); - foreach (string input in inputs) + Parallel.ForEach(inputs, Globals.ParallelOptions, input => { // Add to the list if the input is a directory if (Directory.Exists(input)) { Globals.Logger.Verbose("Adding depot: '" + input + "'"); - directories.Add(input); + lock (directories) + { + directories.Add(input); + } } - } + }); // If we don't have any directories, we want to exit if (directories.Count == 0) diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs b/SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs index a0b18b18..53ff9067 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using System.Web; using SabreTools.Helper.Data; @@ -31,17 +32,23 @@ namespace SabreTools.Helper.Dats { // Make sure all of the extensions have a dot at the beginning List newExtA = new List(); - foreach (string s in extA) + Parallel.ForEach(extA, Globals.ParallelOptions, s => { - newExtA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant()); - } + lock (newExtA) + { + newExtA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant()); + } + }); string newExtAString = string.Join(",", newExtA); List newExtB = new List(); - foreach (string s in extB) + Parallel.ForEach(extB, Globals.ParallelOptions, s => { - newExtB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant()); - } + lock (newExtB) + { + newExtB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant()); + } + }); string newExtBString = string.Join(",", newExtB); // Set all of the appropriate outputs for each of the subsets @@ -83,25 +90,27 @@ namespace SabreTools.Helper.Dats } // Now separate the roms accordingly - foreach (string key in Keys) + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - foreach (DatItem rom in this[key]) + List items = this[key]; + Parallel.ForEach(items, Globals.ParallelOptions, item => { - if (newExtA.Contains(Path.GetExtension(rom.Name.ToUpperInvariant()))) + if (newExtA.Contains(Path.GetExtension(item.Name.ToUpperInvariant()))) { - datdataA.Add(key, rom); + datdataA.Add(key, item); } - else if (newExtB.Contains(Path.GetExtension(rom.Name.ToUpperInvariant()))) + else if (newExtB.Contains(Path.GetExtension(item.Name.ToUpperInvariant()))) { - datdataB.Add(key, rom); + datdataB.Add(key, item); } else { - datdataA.Add(key, rom); - datdataB.Add(key, rom); + datdataA.Add(key, item); + datdataB.Add(key, item); } - } - } + }); + }); // Get the output directory if (outDir != "") @@ -304,65 +313,65 @@ namespace SabreTools.Helper.Dats // Now populate each of the DAT objects in turn List keys = Keys.ToList(); - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - List roms = this[key]; - foreach (DatItem rom in roms) + List items = this[key]; + Parallel.ForEach(items, Globals.ParallelOptions, item => { // If the file is not a Rom or Disk, continue - if (rom.Type != ItemType.Disk && rom.Type != ItemType.Rom) + if (item.Type != ItemType.Disk && item.Type != ItemType.Rom) { - continue; + return; } // If the file is a nodump - if ((rom.Type == ItemType.Rom && ((Rom)rom).ItemStatus == ItemStatus.Nodump) - || (rom.Type == ItemType.Disk && ((Disk)rom).ItemStatus == ItemStatus.Nodump)) + if ((item.Type == ItemType.Rom && ((Rom)item).ItemStatus == ItemStatus.Nodump) + || (item.Type == ItemType.Disk && ((Disk)item).ItemStatus == ItemStatus.Nodump)) { - nodump.Add(key, rom); + nodump.Add(key, item); } // If the file has a SHA-512 - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA512)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA512))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA512)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA512))) { - sha512.Add(key, rom); + sha512.Add(key, item); } // If the file has a SHA-384 - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA384)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA384))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA384)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA384))) { - sha384.Add(key, rom); + sha384.Add(key, item); } // If the file has a SHA-256 - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA256)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA256))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA256)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA256))) { - sha256.Add(key, rom); + sha256.Add(key, item); } // If the file has a SHA-1 - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA1)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA1))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1))) { - sha1.Add(key, rom); + sha1.Add(key, item); } // If the file has no SHA-1 but has an MD5 - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).MD5)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).MD5))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).MD5)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).MD5))) { - md5.Add(key, rom); + md5.Add(key, item); } // If the file has no MD5 but a CRC - else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA1)) - || (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA1))) + else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1)) + || (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1))) { - crc.Add(key, rom); + crc.Add(key, item); } else { - other.Add(key, rom); + other.Add(key, item); } - } - } + }); + }); // Get the output directory if (outDir != "") @@ -415,7 +424,7 @@ namespace SabreTools.Helper.Dats keys.Sort(SplitByLevelSort); // Then, we loop over the games - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Here, the key is the name of the game to be used for comparison if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key)) @@ -440,7 +449,7 @@ namespace SabreTools.Helper.Dats // Then set the DAT name to be the parent directory name tempDat.Name = Style.GetDirectoryName(key); - } + }); // Then we write the last DAT out since it would be skipped otherwise SplitByLevelHelper(tempDat, outDir, shortname, basedat); @@ -581,28 +590,28 @@ namespace SabreTools.Helper.Dats // Now populate each of the DAT objects in turn List keys = Keys.ToList(); - foreach (string key in keys) + Parallel.ForEach(keys, Globals.ParallelOptions, key => { - List roms = this[key]; - foreach (DatItem rom in roms) + List items = this[key]; + Parallel.ForEach(items, Globals.ParallelOptions, item => { // If the file is a Rom - if (rom.Type == ItemType.Rom) + if (item.Type == ItemType.Rom) { - romdat.Add(key, rom); + romdat.Add(key, item); } // If the file is a Disk - else if (rom.Type == ItemType.Disk) + else if (item.Type == ItemType.Disk) { - diskdat.Add(key, rom); + diskdat.Add(key, item); } // If the file is a Sample - else if (rom.Type == ItemType.Sample) + else if (item.Type == ItemType.Sample) { - sampledat.Add(key, rom); + sampledat.Add(key, item); } - } - } + }); + }); // Get the output directory if (outDir != "") diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs b/SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs index c49812d6..e7d40dea 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using System.Web; using SabreTools.Helper.Data; @@ -51,10 +52,11 @@ namespace SabreTools.Helper.Dats } // Loop through and add - foreach (string key in Keys) + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; - foreach (DatItem item in items) + Parallel.ForEach(items, Globals.ParallelOptions, item => { switch (item.Type) { @@ -68,6 +70,8 @@ namespace SabreTools.Helper.Dats MD5Count += (String.IsNullOrEmpty(disk.MD5) ? 0 : 1); SHA1Count += (String.IsNullOrEmpty(disk.SHA1) ? 0 : 1); SHA256Count += (String.IsNullOrEmpty(disk.SHA256) ? 0 : 1); + SHA384Count += (String.IsNullOrEmpty(disk.SHA384) ? 0 : 1); + SHA512Count += (String.IsNullOrEmpty(disk.SHA512) ? 0 : 1); BaddumpCount += (disk.ItemStatus == ItemStatus.BadDump ? 1 : 0); NodumpCount += (disk.ItemStatus == ItemStatus.Nodump ? 1 : 0); break; @@ -81,14 +85,16 @@ namespace SabreTools.Helper.Dats MD5Count += (String.IsNullOrEmpty(rom.MD5) ? 0 : 1); SHA1Count += (String.IsNullOrEmpty(rom.SHA1) ? 0 : 1); SHA256Count += (String.IsNullOrEmpty(rom.SHA256) ? 0 : 1); + SHA384Count += (String.IsNullOrEmpty(rom.SHA384) ? 0 : 1); + SHA512Count += (String.IsNullOrEmpty(rom.SHA512) ? 0 : 1); BaddumpCount += (rom.ItemStatus == ItemStatus.BadDump ? 1 : 0); NodumpCount += (rom.ItemStatus == ItemStatus.Nodump ? 1 : 0); break; case ItemType.Sample: break; } - } - } + }); + }); } /// diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs index 3a06a00f..203a6102 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; using System.Web; using SabreTools.Helper.Data; @@ -134,7 +135,8 @@ namespace SabreTools.Helper.Dats List keys = Keys.ToList(); keys.Sort(new NaturalComparer()); - foreach (DatFormat datFormat in outfiles.Keys) + // Write out all required formats + Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat => { string outfile = outfiles[datFormat]; @@ -229,7 +231,7 @@ namespace SabreTools.Helper.Dats Globals.Logger.Verbose("File written!" + Environment.NewLine); sw.Dispose(); fs.Dispose(); - } + }); } catch (Exception ex) {