[DatFile] Fix some parallelization

2025-12-16 19:14:27 +00:00 · 2017-03-02 12:25:05 -08:00
parent 082599111d
commit eff85278d3
3 changed files with 91 additions and 81 deletions
--- a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs
+++ b/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs
@@ -35,96 +35,92 @@ namespace SabreTools.Helper.Dats
 			// Create the temporary dictionary to sort into
 			SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();

-			Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms by " + bucketBy);
+			Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : ""));

 			// First do the initial sort of all of the roms
 			List<string> keys = Keys.ToList();
 			Parallel.ForEach(keys,
 				new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
 				key =>
-				{
-					List<DatItem> roms = this[key];
+			{
+				List<DatItem> roms = this[key];

-					// If we're merging the roms, do so
-					if (mergeroms)
+				// Now add each of the roms to their respective games
+				Parallel.ForEach(roms,
+					new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
+					rom =>
+				{
+					string newkey = "";
+
+					// We want to get the key most appropriate for the given sorting type
+					switch (bucketBy)
 					{
-						roms = DatItem.Merge(roms);
+						case SortedBy.CRC:
+							newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
+							break;
+						case SortedBy.Game:
+							newkey = (norename ? ""
+								: rom.SystemID.ToString().PadLeft(10, '0')
+									+ "-"
+									+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
+							+ (String.IsNullOrEmpty(rom.Machine.Name)
+									? "Default"
+									: rom.Machine.Name);
+							if (lower)
+							{
+								newkey = newkey.ToLowerInvariant();
+							}
+
+							newkey = HttpUtility.HtmlEncode(newkey);
+							break;
+						case SortedBy.MD5:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).MD5
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).MD5
+									: Constants.MD5Zero));
+							break;
+						case SortedBy.SHA1:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA1
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA1
+									: Constants.SHA1Zero));
+							break;
+						case SortedBy.SHA256:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA256
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA256
+									: Constants.SHA256Zero));
+							break;
+						case SortedBy.SHA384:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA384
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA384
+									: Constants.SHA384Zero));
+							break;
+						case SortedBy.SHA512:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA512
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA512
+									: Constants.SHA512Zero));
+							break;
 					}

-					// Now add each of the roms to their respective games
-					foreach (DatItem rom in roms)
+					// Add the DatItem to the temp dictionary
+					lock (sortable)
 					{
-						string newkey = "";
-
-						// We want to get the key most appropriate for the given sorting type
-						switch (bucketBy)
+						if (!sortable.ContainsKey(newkey))
 						{
-							case SortedBy.CRC:
-								newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
-								break;
-							case SortedBy.Game:
-								newkey = (norename ? ""
-									: rom.SystemID.ToString().PadLeft(10, '0')
-										+ "-"
-										+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
-								+ (String.IsNullOrEmpty(rom.Machine.Name)
-										? "Default"
-										: rom.Machine.Name);
-										if (lower)
-										{
-											newkey = newkey.ToLowerInvariant();
-										}
-
-								newkey = HttpUtility.HtmlEncode(newkey);
-								break;
-							case SortedBy.MD5:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).MD5
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).MD5
-										: Constants.MD5Zero));
-								break;
-							case SortedBy.SHA1:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA1
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA1
-										: Constants.SHA1Zero));
-								break;
-							case SortedBy.SHA256:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA256
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA256
-										: Constants.SHA256Zero));
-								break;
-							case SortedBy.SHA384:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA384
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA384
-										: Constants.SHA384Zero));
-								break;
-							case SortedBy.SHA512:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA512
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA512
-										: Constants.SHA512Zero));
-								break;
-						}
-
-						// Add the DatItem to the temp dictionary
-						lock (sortable)
-						{
-							if (!sortable.ContainsKey(newkey))
-							{
-								sortable.Add(newkey, new List<DatItem>());
-							}
-							sortable[newkey].Add(rom);
+							sortable.Add(newkey, new List<DatItem>());
 						}
+						sortable[newkey].Add(rom);
 					}
 				});
+			});

 			// Now go through and sort all of the individual lists
 			keys = sortable.Keys.ToList();
@@ -132,9 +128,19 @@ namespace SabreTools.Helper.Dats
 				new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
 				key =>
 			{
+				// Get the possibly unsorted list
 				List<DatItem> sortedlist = sortable[key];
+
+				// Sort the list of items to be consistent
 				DatItem.Sort(ref sortedlist, false);

+				// If we're merging the roms, do so
+				if (mergeroms)
+				{
+					sortedlist = DatItem.Merge(sortedlist);
+				}
+
+				// Add the list back to the temp dictionary
 				lock (sortable)
 				{
 					sortable[key] = sortedlist;
--- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs
+++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs
@@ -100,8 +100,7 @@ namespace SabreTools.Helper.Dats
 			DateTime start = DateTime.Now;
 			Globals.Logger.User("Processing individual DATs");

-			// TODO: Can parsing headers be separated from parsing content?
-			// TODO: Can all DATs be parsed into the same structure in one loop?
+			// Parse all of the DATs into their own DatFiles in the array
 			Parallel.For(0,
 				inputs.Count,
 				new ParallelOptions { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
--- a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
+++ b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
@@ -38,8 +38,7 @@ namespace SabreTools.Helper.Dats
 		/// The following features have been requested for file output:
 		/// - Have the ability to strip special (non-ASCII) characters from rom information
 		/// </remarks>
-		public bool WriteToFile(string outDir,
-			bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
+		public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
 		{
 			// If there's nothing there, abort
 			if (Count == 0)
@@ -109,8 +108,14 @@ namespace SabreTools.Helper.Dats
 					recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true);
 			}

-			// Bucket roms by game name and optionally dedupe
-			BucketBy(SortedBy.Game, MergeRoms, norename: norename);
+			// First bucket by CRC to dedupe if required
+			if (MergeRoms)
+			{
+				BucketBy(SortedBy.CRC, MergeRoms, norename: norename);
+			}
+
+			// Bucket roms by game name
+			BucketBy(SortedBy.Game, false /* mergeRoms */, norename: norename);

 			// Output the number of items we're going to be writing
 			Globals.Logger.User("A total of " + Count + " items will be written out to file");