[DatFile] Fix some parallelization

SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs

@@ -35,7 +35,7 @@ namespace SabreTools.Helper.Dats
 			// Create the temporary dictionary to sort into
 			SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
 
-			Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms by " + bucketBy);
+			Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : ""));
 
 			// First do the initial sort of all of the roms
 			List<string> keys = Keys.ToList();
@@ -45,14 +45,10 @@ namespace SabreTools.Helper.Dats
 			{
 				List<DatItem> roms = this[key];
 
-					// If we're merging the roms, do so
-					if (mergeroms)
-					{
-						roms = DatItem.Merge(roms);
-					}
-
 				// Now add each of the roms to their respective games
-					foreach (DatItem rom in roms)
+				Parallel.ForEach(roms,
+					new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
+					rom =>
 				{
 					string newkey = "";
 
@@ -123,7 +119,7 @@ namespace SabreTools.Helper.Dats
 						}
 						sortable[newkey].Add(rom);
 					}
-					}
+				});
 			});
 
 			// Now go through and sort all of the individual lists
@@ -132,9 +128,19 @@ namespace SabreTools.Helper.Dats
 				new ParallelOptions() { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },
 				key =>
 			{
+				// Get the possibly unsorted list
 				List<DatItem> sortedlist = sortable[key];
+
+				// Sort the list of items to be consistent
 				DatItem.Sort(ref sortedlist, false);
 
+				// If we're merging the roms, do so
+				if (mergeroms)
+				{
+					sortedlist = DatItem.Merge(sortedlist);
+				}
+
+				// Add the list back to the temp dictionary
 				lock (sortable)
 				{
 					sortable[key] = sortedlist;

SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs

@@ -100,8 +100,7 @@ namespace SabreTools.Helper.Dats
 			DateTime start = DateTime.Now;
 			Globals.Logger.User("Processing individual DATs");
 
-			// TODO: Can parsing headers be separated from parsing content?
-			// TODO: Can all DATs be parsed into the same structure in one loop?
+			// Parse all of the DATs into their own DatFiles in the array
 			Parallel.For(0,
 				inputs.Count,
 				new ParallelOptions { MaxDegreeOfParallelism = Globals.MaxDegreeOfParallelism },

SabreTools.Helper/Dats/Partials/DatFile.Writers.cs

@@ -38,8 +38,7 @@ namespace SabreTools.Helper.Dats
 		/// The following features have been requested for file output:
 		/// - Have the ability to strip special (non-ASCII) characters from rom information
 		/// </remarks>
-		public bool WriteToFile(string outDir,
-			bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
+		public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
 		{
 			// If there's nothing there, abort
 			if (Count == 0)
@@ -109,8 +108,14 @@ namespace SabreTools.Helper.Dats
 					recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true);
 			}
 
-			// Bucket roms by game name and optionally dedupe
-			BucketBy(SortedBy.Game, MergeRoms, norename: norename);
+			// First bucket by CRC to dedupe if required
+			if (MergeRoms)
+			{
+				BucketBy(SortedBy.CRC, MergeRoms, norename: norename);
+			}
+
+			// Bucket roms by game name
+			BucketBy(SortedBy.Game, false /* mergeRoms */, norename: norename);
 
 			// Output the number of items we're going to be writing
 			Globals.Logger.User("A total of " + Count + " items will be written out to file");