[DatFile] Make descAsName a separate method

Making this separate allows for updating of cloneof/romof/sampleof tags at the same time and in parallel. This change also fully hooks up the remove unicode flag, though the flag itself has some issues.

SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs

@@ -32,6 +32,7 @@ namespace SabreTools.Helper.Dats
 		/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
 		/// <param name="bare">True if the date should not be appended to the default name, false otherwise [OBSOLETE]</param>
 		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
 		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
 		/// <param name="filter">Filter object to be passed to the DatItem level</param>
 		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
@@ -39,7 +40,7 @@ namespace SabreTools.Helper.Dats
 		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
 		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		public void DetermineUpdateType(List<string> inputPaths, string outDir, bool merge, DiffMode diff, bool inplace, bool skip,
-			bool bare, bool clean, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root)
+			bool bare, bool clean, bool remUnicode, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root)
 		{
 			// If we're in merging or diffing mode, use the full list of inputs
 			if (merge || diff != 0)
@@ -54,8 +55,8 @@ namespace SabreTools.Helper.Dats
 				}
 
 				// Create a dictionary of all ROMs from the input DATs
-				List<DatFile> datHeaders = PopulateUserData(newInputFileNames, inplace, clean, descAsName,
+				List<DatFile> datHeaders = PopulateUserData(newInputFileNames, inplace, clean,
-					outDir, filter, splitType, trim, single, root);
+					remUnicode, descAsName, outDir, filter, splitType, trim, single, root);
 
 				// Modify the Dictionary if necessary and output the results
 				if (diff != 0 && diff < DiffMode.Cascade)
@@ -76,7 +77,7 @@ namespace SabreTools.Helper.Dats
 			// Otherwise, loop through all of the inputs individually
 			else
 			{
-				Update(inputPaths, outDir, clean, descAsName, filter, splitType, trim, single, root);
+				Update(inputPaths, outDir, clean, remUnicode, descAsName, filter, splitType, trim, single, root);
 			}
 			return;
 		}
@@ -90,8 +91,8 @@ namespace SabreTools.Helper.Dats
 		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
 		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <returns>List of DatData objects representing headers</returns>
-		private List<DatFile> PopulateUserData(List<string> inputs, bool inplace, bool clean, bool descAsName, string outDir,
+		private List<DatFile> PopulateUserData(List<string> inputs, bool inplace, bool clean, bool remUnicode, bool descAsName,
-			Filter filter, SplitType splitType, bool trim, bool single, string root)
+			string outDir, Filter filter, SplitType splitType, bool trim, bool single, string root)
 		{
 			DatFile[] datHeaders = new DatFile[inputs.Count];
 			DateTime start = DateTime.Now;
@@ -518,14 +519,15 @@ namespace SabreTools.Helper.Dats
 		/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
 		/// <param name="bare">True if the date should not be appended to the default name, false otherwise [OBSOLETE]</param>
 		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
 		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
 		/// <param name="filter">Filter object to be passed to the DatItem level</param>
 		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
 		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
 		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
 		/// <param name="root">String representing root directory to compare against for length calculation</param>
-		public void Update(List<string> inputFileNames, string outDir, bool clean, bool descAsName, Filter filter,
+		public void Update(List<string> inputFileNames, string outDir, bool clean, bool remUnicode, bool descAsName,
-			SplitType splitType, bool trim, bool single, string root)
+			Filter filter, SplitType splitType, bool trim, bool single, string root)
 		{
 			Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName =>
 			{
@@ -539,7 +541,7 @@ namespace SabreTools.Helper.Dats
 				{
 					DatFile innerDatdata = new DatFile(this);
 					Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
-					innerDatdata.Parse(inputFileName, 0, 0, splitType, true, clean, descAsName,
+					innerDatdata.Parse(inputFileName, 0, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName,
 						keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
 					innerDatdata.Filter(filter, trim, single, root);
 

SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs

@@ -208,6 +208,82 @@ namespace SabreTools.Helper.Dats
 			});
 		}
 
+		/// <summary>
+		/// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof
+		/// </summary>
+		public void MachineDescriptionToName()
+		{
+			try
+			{
+				// First we want to get a mapping for all games to description
+				Dictionary<string, string> mapping = new Dictionary<string, string>();
+				List<string> keys = Keys.ToList();
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+				{
+					List<DatItem> items = this[key];
+					Parallel.ForEach(items, Globals.ParallelOptions, item =>
+					{
+						lock (mapping)
+						{
+							// If the key mapping doesn't exist, add it
+							if (!mapping.ContainsKey(item.Machine.Name))
+							{
+								mapping.Add(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''"));
+							}
+						}
+					});
+				});
+
+				// Now we loop through every item and update accordingly
+				keys = Keys.ToList();
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+				{
+					List<DatItem> items = this[key];
+					List<DatItem> newItems = new List<DatItem>();
+					Parallel.ForEach(items, Globals.ParallelOptions, item =>
+					{
+						// Update machine name
+						if (item.Machine.Name != null && mapping.ContainsKey(item.Machine.Name))
+						{
+							item.Machine.Name = mapping[item.Machine.Name];
+						}
+
+						// Update cloneof
+						if (item.Machine.CloneOf != null && mapping.ContainsKey(item.Machine.CloneOf))
+						{
+							item.Machine.CloneOf = mapping[item.Machine.CloneOf];
+						}
+
+						// Update romof
+						if (item.Machine.RomOf != null && mapping.ContainsKey(item.Machine.RomOf))
+						{
+							item.Machine.RomOf = mapping[item.Machine.RomOf];
+						}
+
+						// Update sampleof
+						if (item.Machine.SampleOf != null && mapping.ContainsKey(item.Machine.SampleOf))
+						{
+							item.Machine.SampleOf = mapping[item.Machine.SampleOf];
+						}
+
+						// Add the new item to the output list
+						lock (newItems)
+						{
+							newItems.Add(item);
+						}
+					});
+
+					// Replace the old list of roms with the new one
+					Remove(key);
+					AddRange(key, newItems);
+				});
+			}
+			catch (Exception ex)
+			{
+				Globals.Logger.Warning(ex.ToString());
+			}
+		}
+
 #endregion
 
 #region Merging/Splitting Methods
@@ -890,68 +966,5 @@ namespace SabreTools.Helper.Dats
 #endregion
 
 #endregion // Instance Methods
-
-#region Static Methods
-
-#region Bucketing
-
-		/// <summary>
-		/// Take an arbitrarily ordered List and return a Dictionary sorted by Game
-		/// </summary>
-		/// <param name="list">Input unsorted list</param>
-		/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
-		/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
-		/// <param name="output">True if the number of hashes counted is to be output (default), false otherwise</param>
-		/// <returns>SortedDictionary bucketed by game name</returns>
-		public static SortedDictionary<string, List<DatItem>> BucketListByGame(List<DatItem> list, bool mergeroms, bool norename, bool output = true)
-		{
-			Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output");
-
-			SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
-			long count = 0;
-
-			// If we have a null dict or an empty one, output a new dictionary
-			if (list == null || list.Count == 0)
-			{
-				return sortable;
-			}
-
-			// If we're merging the roms, do so
-			if (mergeroms)
-			{
-				list = DatItem.Merge(list);
-			}
-
-			// Now add each of the roms to their respective games
-			foreach (DatItem rom in list)
-			{
-				if (rom == null)
-				{
-					continue;
-				}
-
-				count++;
-				string newkey = (norename ? ""
-						: rom.SystemID.ToString().PadLeft(10, '0')
-							+ "-"
-							+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
-					+ (rom.Machine == null || String.IsNullOrEmpty(rom.Machine.Name)
-							? "Default"
-							: rom.Machine.Name.ToLowerInvariant());
-				newkey = HttpUtility.HtmlEncode(newkey);
-
-				if (!sortable.ContainsKey(newkey))
-				{
-					sortable.Add(newkey, new List<DatItem>());
-				}
-				sortable[newkey].Add(rom);
-			}
-
-			return sortable;
-		}
-
-#endregion
-
-#endregion // Static Methods
 	}
 }

SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs

@@ -55,8 +55,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
 		/// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param>
-		/// TODO: If replacing name with description, is it possible to update cloneof/romof/sampleof tags as well?
-		/// TODO: If replacing name with description, should this be done in a separate step instead of on parse? It might help the other task
 		public void Parse(
 			// Standard Dat parsing
 			string filename,
@@ -98,20 +96,20 @@ namespace SabreTools.Helper.Dats
 				switch (FileTools.GetDatFormat(filename))
 				{
 					case DatFormat.AttractMode:
-						ParseAttractMode(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
+						ParseAttractMode(filename, sysid, srcid, keep, clean, remUnicode);
 						break;
 					case DatFormat.ClrMamePro:
 					case DatFormat.DOSCenter:
-						ParseCMP(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
+						ParseCMP(filename, sysid, srcid, keep, clean, remUnicode);
 						break;
 					case DatFormat.CSV:
-						ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, remUnicode, descAsName);
+						ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, remUnicode);
 						break;
 					case DatFormat.Logiqx:
 					case DatFormat.OfflineList:
 					case DatFormat.SabreDat:
 					case DatFormat.SoftwareList:
-						ParseGenericXML(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
+						ParseGenericXML(filename, sysid, srcid, keep, clean, remUnicode);
 						break;
 					case DatFormat.RedumpMD5:
 						ParseRedumpMD5(filename, sysid, srcid, clean, remUnicode);
@@ -132,10 +130,10 @@ namespace SabreTools.Helper.Dats
 						ParseRedumpSHA512(filename, sysid, srcid, clean, remUnicode);
 						break;
 					case DatFormat.RomCenter:
-						ParseRC(filename, sysid, srcid, clean, remUnicode, descAsName);
+						ParseRC(filename, sysid, srcid, clean, remUnicode);
 						break;
 					case DatFormat.TSV:
-						ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, remUnicode, descAsName);
+						ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, remUnicode);
 						break;
 					default:
 						return;
@@ -146,6 +144,12 @@ namespace SabreTools.Helper.Dats
 				Globals.Logger.Error("Error with file '" + filename + "': " + ex.ToString());
 			}
 
+			// If we want to use descriptions as names, update everything
+			if (descAsName)
+			{
+				MachineDescriptionToName();
+			}
+
 			// If we are using tags from the DAT, set the proper input for split type unless overridden
 			if (useTags && splitType == SplitType.None)
 			{
@@ -196,7 +200,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
-		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		private void ParseAttractMode(
 			// Standard Dat parsing
 			string filename,
@@ -206,8 +209,7 @@ namespace SabreTools.Helper.Dats
 			// Miscellaneous
 			bool keep,
 			bool clean,
-			bool remUnicode,
+			bool remUnicode)
-			bool descAsName)
 		{
 			// Open a file reader
 			Encoding enc = Style.GetEncoding(filename);
@@ -252,7 +254,7 @@ namespace SabreTools.Helper.Dats
 
 					Machine = new Machine
 					{
-						Name = (descAsName ? gameinfo[1] : gameinfo[0]),
+						Name = gameinfo[0],
 						Description = gameinfo[1],
 						CloneOf = gameinfo[3],
 						Year = gameinfo[4],
@@ -277,7 +279,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
-		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		private void ParseCMP(
 			// Standard Dat parsing
 			string filename,
@@ -287,8 +288,7 @@ namespace SabreTools.Helper.Dats
 			// Miscellaneous
 			bool keep,
 			bool clean,
-			bool remUnicode,
+			bool remUnicode)
-			bool descAsName)
 		{
 			// Open a file reader
 			Encoding enc = Style.GetEncoding(filename);
@@ -712,12 +712,6 @@ namespace SabreTools.Helper.Dats
 								break;
 							case "description":
 								gamedesc = itemval;
-
-								// If we want to have the description as the name, do so
-								if (descAsName)
-								{
-									tempgamename = gamedesc.Replace('/', '_').Replace("\"", "''");
-								}
 								break;
 							case "romof":
 								romof = itemval;
@@ -882,7 +876,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
-		/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
 		private void ParseCSVTSV(
 			// Standard Dat parsing
 			string filename,
@@ -893,8 +886,7 @@ namespace SabreTools.Helper.Dats
 			// Miscellaneous
 			bool keep,
 			bool clean,
-			bool remUnicode,
+			bool remUnicode)
-			bool descAsName)
 		{
 			// Open a file reader
 			Encoding enc = Style.GetEncoding(filename);
@@ -1035,7 +1027,6 @@ namespace SabreTools.Helper.Dats
 							break;
 						case "Machine.Description":
 							machineDesc = value;
-							machineName = (descAsName ? value : machineName);
 							break;
 						case "DatItem.Type":
 							switch (value.ToLowerInvariant())
@@ -1230,7 +1221,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
-		/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
 		/// <remrks>
 		/// TODO: Software Lists - sharedfeat tag (read-in, write-out)
 		/// </remrks>
@@ -1243,8 +1233,7 @@ namespace SabreTools.Helper.Dats
 			// Miscellaneous
 			bool keep,
 			bool clean,
-			bool remUnicode,
+			bool remUnicode)
-			bool descAsName)
 		{
 			// Prepare all internal variables
 			XmlReader subreader, headreader, flagreader;
@@ -1877,12 +1866,6 @@ namespace SabreTools.Helper.Dats
 									// For Logiqx, SabreDAT, and Software List
 									case "description":
 										machine.Description = subreader.ReadElementContentAsString();
-
-										// If we want to have the description as the name, do so
-										if (descAsName)
-										{
-											machine.Name = machine.Description.Replace('/', '_').Replace("\"", "''");
-										}
 										break;
 									case "year":
 										machine.Year = subreader.ReadElementContentAsString();
@@ -2667,7 +2650,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="srcid">Source ID for the DAT</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
-		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		private void ParseRC(
 			// Standard Dat parsing
 			string filename,
@@ -2676,8 +2658,7 @@ namespace SabreTools.Helper.Dats
 
 			// Miscellaneous
 			bool clean,
-			bool remUnicode,
+			bool remUnicode)
-			bool descAsName)
 		{
 			// Open a file reader
 			Encoding enc = Style.GetEncoding(filename);
@@ -2804,7 +2785,7 @@ namespace SabreTools.Helper.Dats
 
 							Machine = new Machine
 							{
-								Name = (descAsName ? rominfo[4] : rominfo[3]),
+								Name = rominfo[3],
 								Description = rominfo[4],
 								CloneOf = rominfo[1],
 								RomOf = rominfo[8],

SabreTools/Partials/SabreTools.Inits.cs

@@ -700,8 +700,8 @@ namespace SabreTools
 				Romba = romba,
 			};
 			
-			userInputDat.DetermineUpdateType(inputs, outDir, merge, diffMode, inplace, skip, bare, clean, descAsName,
+			userInputDat.DetermineUpdateType(inputs, outDir, merge, diffMode, inplace, skip, bare, clean,
-				filter, splitType, trim, single, root);
+				remUnicode, descAsName, filter, splitType, trim, single, root);
 		}
 
 		/// <summary>