[DatFile] Fix some parallelization

SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs

@@ -35,96 +35,92 @@ namespace SabreTools.Helper.Dats
 			// Create the temporary dictionary to sort into
 			SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
 
-			Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms by " + bucketBy);
+			Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : ""));
 
 			// First do the initial sort of all of the roms
 			List<string> keys = Keys.ToList();
 			Parallel.ForEach(keys,
 				new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
 				key =>
-				{
+			{
-					List<DatItem> roms = this[key];
+				List<DatItem> roms = this[key];
 
-					// If we're merging the roms, do so
+				// Now add each of the roms to their respective games
-					if (mergeroms)
+				Parallel.ForEach(roms,
+					new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
+					rom =>
+				{
+					string newkey = "";
+
+					// We want to get the key most appropriate for the given sorting type
+					switch (bucketBy)
 					{
-						roms = DatItem.Merge(roms);
+						case SortedBy.CRC:
+							newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
+							break;
+						case SortedBy.Game:
+							newkey = (norename ? ""
+								: rom.SystemID.ToString().PadLeft(10, '0')
+									+ "-"
+									+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
+							+ (String.IsNullOrEmpty(rom.Machine.Name)
+									? "Default"
+									: rom.Machine.Name);
+							if (lower)
+							{
+								newkey = newkey.ToLowerInvariant();
+							}
+
+							newkey = HttpUtility.HtmlEncode(newkey);
+							break;
+						case SortedBy.MD5:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).MD5
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).MD5
+									: Constants.MD5Zero));
+							break;
+						case SortedBy.SHA1:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA1
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA1
+									: Constants.SHA1Zero));
+							break;
+						case SortedBy.SHA256:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA256
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA256
+									: Constants.SHA256Zero));
+							break;
+						case SortedBy.SHA384:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA384
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA384
+									: Constants.SHA384Zero));
+							break;
+						case SortedBy.SHA512:
+							newkey = (rom.Type == ItemType.Rom
+								? ((Rom)rom).SHA512
+								: (rom.Type == ItemType.Disk
+									? ((Disk)rom).SHA512
+									: Constants.SHA512Zero));
+							break;
 					}
 
-					// Now add each of the roms to their respective games
+					// Add the DatItem to the temp dictionary
-					foreach (DatItem rom in roms)
+					lock (sortable)
 					{
-						string newkey = "";
+						if (!sortable.ContainsKey(newkey))
-
-						// We want to get the key most appropriate for the given sorting type
-						switch (bucketBy)
 						{
-							case SortedBy.CRC:
+							sortable.Add(newkey, new List<DatItem>());
-								newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
-								break;
-							case SortedBy.Game:
-								newkey = (norename ? ""
-									: rom.SystemID.ToString().PadLeft(10, '0')
-										+ "-"
-										+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
-								+ (String.IsNullOrEmpty(rom.Machine.Name)
-										? "Default"
-										: rom.Machine.Name);
-										if (lower)
-										{
-											newkey = newkey.ToLowerInvariant();
-										}
-
-								newkey = HttpUtility.HtmlEncode(newkey);
-								break;
-							case SortedBy.MD5:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).MD5
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).MD5
-										: Constants.MD5Zero));
-								break;
-							case SortedBy.SHA1:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA1
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA1
-										: Constants.SHA1Zero));
-								break;
-							case SortedBy.SHA256:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA256
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA256
-										: Constants.SHA256Zero));
-								break;
-							case SortedBy.SHA384:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA384
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA384
-										: Constants.SHA384Zero));
-								break;
-							case SortedBy.SHA512:
-								newkey = (rom.Type == ItemType.Rom
-									? ((Rom)rom).SHA512
-									: (rom.Type == ItemType.Disk
-										? ((Disk)rom).SHA512
-										: Constants.SHA512Zero));
-								break;
-						}
-
-						// Add the DatItem to the temp dictionary
-						lock (sortable)
-						{
-							if (!sortable.ContainsKey(newkey))
-							{
-								sortable.Add(newkey, new List<DatItem>());
-							}
-							sortable[newkey].Add(rom);
 						}
+						sortable[newkey].Add(rom);
 					}
 				});
+			});
 
 			// Now go through and sort all of the individual lists
 			keys = sortable.Keys.ToList();
@@ -132,9 +128,19 @@ namespace SabreTools.Helper.Dats
 				new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
 				key =>
 			{
+				// Get the possibly unsorted list
 				List<DatItem> sortedlist = sortable[key];
+
+				// Sort the list of items to be consistent
 				DatItem.Sort(ref sortedlist, false);
 
+				// If we're merging the roms, do so
+				if (mergeroms)
+				{
+					sortedlist = DatItem.Merge(sortedlist);
+				}
+
+				// Add the list back to the temp dictionary
 				lock (sortable)
 				{
 					sortable[key] = sortedlist;

SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs

@@ -100,8 +100,7 @@ namespace SabreTools.Helper.Dats
 			DateTime start = DateTime.Now;
 			Globals.Logger.User("Processing individual DATs");
 
-			// TODO: Can parsing headers be separated from parsing content?
+			// Parse all of the DATs into their own DatFiles in the array
-			// TODO: Can all DATs be parsed into the same structure in one loop?
 			Parallel.For(0,
 				inputs.Count,
 				new ParallelOptions { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },

SabreTools.Helper/Dats/Partials/DatFile.Writers.cs

@@ -38,8 +38,7 @@ namespace SabreTools.Helper.Dats
 		/// The following features have been requested for file output:
 		/// - Have the ability to strip special (non-ASCII) characters from rom information
 		/// </remarks>
-		public bool WriteToFile(string outDir,
+		public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
-			bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
 		{
 			// If there's nothing there, abort
 			if (Count == 0)
@@ -109,8 +108,14 @@ namespace SabreTools.Helper.Dats
 					recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true);
 			}
 
-			// Bucket roms by game name and optionally dedupe
+			// First bucket by CRC to dedupe if required
-			BucketBy(SortedBy.Game, MergeRoms, norename: norename);
+			if (MergeRoms)
+			{
+				BucketBy(SortedBy.CRC, MergeRoms, norename: norename);
+			}
+
+			// Bucket roms by game name
+			BucketBy(SortedBy.Game, false /* mergeRoms */, norename: norename);
 
 			// Output the number of items we're going to be writing
 			Globals.Logger.User("A total of " + Count + " items will be written out to file");