[DatFile] Make filtering a separate step

Making filter "optional" means that it can be done with multithreading, thus reducing the time that any given DAT takes to parse in most cases and improving the actual filtering in the process.

SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs

@@ -111,22 +111,17 @@ namespace SabreTools.Helper.Dats
 						MergeRoms = MergeRoms,
 					};
 
-					datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single,
+					datHeaders[i].Parse(input.Split('¬')[0], i, 0, splitType, true, clean, descAsName);
-						root, true, clean, descAsName);
 				});
 
 			Globals.Logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
 			Globals.Logger.User("Populating internal DAT");
-			Parallel.For(0, inputs.Count,
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
-				Globals.ParallelOptions,
-				i =>
 			{
 				// Get the list of keys from the DAT
 				List<string> keys = datHeaders[i].Keys.ToList();
-				Parallel.ForEach(keys,
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
-					Globals.ParallelOptions,
-					key =>
 				{
 					// Add everything from the key to the internal DAT
 					AddRange(key, datHeaders[i][key]);
@@ -142,6 +137,9 @@ namespace SabreTools.Helper.Dats
 				datHeaders[i].Delete();
 			});
 
+			// Now that we have a merged DAT, filter it
+			Filter(filter, single, trim, root);
+
 			Globals.Logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
 			return datHeaders.ToList();
@@ -532,9 +530,7 @@ namespace SabreTools.Helper.Dats
 		public void Update(List<string> inputFileNames, string outDir, bool clean, bool descAsName, Filter filter,
 			SplitType splitType, bool trim, bool single, string root)
 		{
-			Parallel.ForEach(inputFileNames,
+			Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName =>
-				Globals.ParallelOptions,
-				inputFileName =>
 			{
 				// Clean the input string
 				if (inputFileName != "")
@@ -546,9 +542,9 @@ namespace SabreTools.Helper.Dats
 				{
 					DatFile innerDatdata = new DatFile(this);
 					Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
-						innerDatdata.Parse(inputFileName, 0, 0, filter, splitType, trim, single,
+					innerDatdata.Parse(inputFileName, 0, 0, splitType, true, clean, descAsName,
-							root, true, clean, descAsName,
 						keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
+					innerDatdata.Filter(filter, trim, single, root);
 
 					// Try to output the file
 					innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), overwrite: (outDir != ""));
@@ -557,15 +553,14 @@ namespace SabreTools.Helper.Dats
 				{
 					inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
 
-						Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories),
+					List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList();
-							Globals.ParallelOptions,
+					Parallel.ForEach(subFiles, Globals.ParallelOptions, file =>
-							file =>
 					{
 						Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
 						DatFile innerDatdata = new DatFile(this);
-								innerDatdata.Parse(file, 0, 0, filter, splitType,
+						innerDatdata.Parse(file, 0, 0, splitType, true, clean, descAsName,
-									trim, single, root, true, clean, descAsName,
 							keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
+						innerDatdata.Filter(filter, trim, single, root);
 
 						// Try to output the file
 						innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)),

SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs

@@ -6,13 +6,19 @@ using System.Web;
 
 using SabreTools.Helper.Data;
 
+#if MONO
+using System.IO;
+#else
+using Alphaleonis.Win32.Filesystem;
+#endif
+
 namespace SabreTools.Helper.Dats
 {
 	public partial class DatFile
 	{
-		#region Instance Methods
+#region Instance Methods
 
-		#region Bucketing [MODULAR DONE]
+#region Bucketing [MODULAR DONE]
 
 		/// <summary>
 		/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
@@ -151,9 +157,62 @@ namespace SabreTools.Helper.Dats
 			_files = sortable;
 		}
 
-		#endregion
+#endregion
 
-		#region Merging/Splitting Methods [MODULAR DONE]
+#region Filtering
+
+		/// <summary>
+		/// Filter a DAT based on input parameters and modify the items
+		/// </summary>
+		/// <param name="filter">Filter object for passing to the DatItem level</param>
+		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		/// <param name="root">String representing root directory to compare against for length calculation</param>
+		public void Filter(Filter filter, bool single, bool trim, string root)
+		{
+			// Loop over every key in the dictionary
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				// For every item in the current key
+				List<DatItem> items = this[key];
+				List<DatItem> newitems = new List<DatItem>();
+				Parallel.ForEach(items, Globals.ParallelOptions, item =>
+				{
+					// If the rom passes the filter, include it
+					if (filter.ItemPasses(item))
+					{
+						// If we are in single game mode, rename all games
+						if (single)
+						{
+							item.Machine.Name = "!";
+						}
+
+						// If we are in NTFS trim mode, trim the game name
+						if (trim)
+						{
+							// Windows max name length is 260
+							int usableLength = 260 - item.Machine.Name.Length - root.Length;
+							if (item.Name.Length > usableLength)
+							{
+								string ext = Path.GetExtension(item.Name);
+								item.Name = item.Name.Substring(0, usableLength - ext.Length);
+								item.Name += ext;
+							}
+						}
+
+						newitems.Add(item);
+					}
+				});
+
+				Remove(key);
+				AddRange(key, newitems);
+			});
+		}
+
+#endregion
+
+#region Merging/Splitting Methods [MODULAR DONE]
 
 		/// <summary>
 		/// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets
@@ -247,9 +306,9 @@ namespace SabreTools.Helper.Dats
 			RemoveTagsFromChild();
 		}
 
-		#endregion
+#endregion
 
-		#region Merging/Splitting Helper Methods [MODULAR DONE]
+#region Merging/Splitting Helper Methods [MODULAR DONE]
 
 		/// <summary>
 		/// Use romof tags to add roms to the children
@@ -830,13 +889,13 @@ namespace SabreTools.Helper.Dats
 			}
 		}
 
-		#endregion
+#endregion
 
-		#endregion // Instance Methods
+#endregion // Instance Methods
 
-		#region Static Methods
+#region Static Methods
 
-		#region Bucketing [MODULAR DONE]
+#region Bucketing [MODULAR DONE]
 
 		/// <summary>
 		/// Take an arbitrarily ordered List and return a Dictionary sorted by Game
@@ -893,8 +952,8 @@ namespace SabreTools.Helper.Dats
 			return sortable;
 		}
 
-		#endregion
+#endregion
 
-		#endregion // Static Methods
+#endregion // Static Methods
 	}
 }

SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs

@@ -34,11 +34,9 @@ namespace SabreTools.Helper.Dats
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
 		/// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param>
-		public void Parse(string filename, int sysid, int srcid,
+		public void Parse(string filename, int sysid, int srcid, bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false)
-			bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false)
 		{
-			Parse(filename, sysid, srcid, new Filter(), SplitType.None, false, false, "",
+			Parse(filename, sysid, srcid, SplitType.None, keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags);
-				keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags);
 		}
 
 		/// <summary>
@@ -46,12 +44,8 @@ namespace SabreTools.Helper.Dats
 		/// </summary>
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
-		/// <param name="srcid">Source ID for the DAT</param>
+		/// <param name="srcid">Source ID for the DAT</param>>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
 		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -63,14 +57,8 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
 			// Rom renaming
 			SplitType splitType,
-			bool trim,
-			bool single,
-			string root,
 
 			// Miscellaneous
 			bool keep = false,
@@ -103,44 +91,44 @@ namespace SabreTools.Helper.Dats
 				switch (FileTools.GetDatFormat(filename))
 				{
 					case DatFormat.AttractMode:
-						ParseAttractMode(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
+						ParseAttractMode(filename, sysid, srcid, keep, clean, descAsName);
 						break;
 					case DatFormat.ClrMamePro:
 					case DatFormat.DOSCenter:
-						ParseCMP(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
+						ParseCMP(filename, sysid, srcid, keep, clean, descAsName);
 						break;
 					case DatFormat.CSV:
-						ParseCSVTSV(filename, sysid, srcid, ',', filter, trim, single, root, keep, clean, descAsName);
+						ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, descAsName);
 						break;
 					case DatFormat.Logiqx:
 					case DatFormat.OfflineList:
 					case DatFormat.SabreDat:
 					case DatFormat.SoftwareList:
-						ParseGenericXML(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
+						ParseGenericXML(filename, sysid, srcid, keep, clean, descAsName);
 						break;
 					case DatFormat.RedumpMD5:
-						ParseRedumpMD5(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpMD5(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RedumpSFV:
-						ParseRedumpSFV(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpSFV(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RedumpSHA1:
-						ParseRedumpSHA1(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpSHA1(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RedumpSHA256:
-						ParseRedumpSHA256(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpSHA256(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RedumpSHA384:
-						ParseRedumpSHA384(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpSHA384(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RedumpSHA512:
-						ParseRedumpSHA512(filename, sysid, srcid, filter, trim, single, root, clean);
+						ParseRedumpSHA512(filename, sysid, srcid, clean);
 						break;
 					case DatFormat.RomCenter:
-						ParseRC(filename, sysid, srcid, filter, trim, single, root, clean, descAsName);
+						ParseRC(filename, sysid, srcid, clean, descAsName);
 						break;
 					case DatFormat.TSV:
-						ParseCSVTSV(filename, sysid, srcid, '\t', filter, trim, single, root, keep, clean, descAsName);
+						ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, descAsName);
 						break;
 					default:
 						return;
@@ -198,10 +186,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -211,14 +195,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool keep,
 			bool clean,
@@ -277,7 +253,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -289,10 +265,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -302,14 +274,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool keep,
 			bool clean,
@@ -421,7 +385,7 @@ namespace SabreTools.Helper.Dats
 
 						// Now process and add the sample
 						key = "";
-						ParseAddHelper(item, filter, trim, single, root, clean, out key);
+						ParseAddHelper(item, clean, out key);
 
 						continue;
 					}
@@ -506,7 +470,7 @@ namespace SabreTools.Helper.Dats
 
 						// Now process and add the rom
 						key = "";
-						ParseAddHelper(item, filter, trim, single, root, clean, out key);
+						ParseAddHelper(item, clean, out key);
 						continue;
 					}
 
@@ -719,7 +683,7 @@ namespace SabreTools.Helper.Dats
 
 					// Now process and add the rom
 					key = "";
-					ParseAddHelper(item, filter, trim, single, root, clean, out key);
+					ParseAddHelper(item, clean, out key);
 				}
 
 				// If the line is anything but a rom or disk and we're in a block
@@ -904,10 +868,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
 		/// <param name="delim">Delimiter for parsing individual lines</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
@@ -918,14 +878,6 @@ namespace SabreTools.Helper.Dats
 			int srcid,
 			char delim,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool keep,
 			bool clean,
@@ -1164,7 +1116,7 @@ namespace SabreTools.Helper.Dats
 							},
 						};
 
-						ParseAddHelper(archive, filter, trim, single, root, clean, out key);
+						ParseAddHelper(archive, clean, out key);
 						break;
 					case ItemType.BiosSet:
 						BiosSet biosset = new BiosSet()
@@ -1178,7 +1130,7 @@ namespace SabreTools.Helper.Dats
 							},
 						};
 
-						ParseAddHelper(biosset, filter, trim, single, root, clean, out key);
+						ParseAddHelper(biosset, clean, out key);
 						break;
 					case ItemType.Disk:
 						Disk disk = new Disk()
@@ -1199,7 +1151,7 @@ namespace SabreTools.Helper.Dats
 							ItemStatus = status,
 						};
 
-						ParseAddHelper(disk, filter, trim, single, root, clean, out key);
+						ParseAddHelper(disk, clean, out key);
 						break;
 					case ItemType.Release:
 						Release release = new Release()
@@ -1213,7 +1165,7 @@ namespace SabreTools.Helper.Dats
 							},
 						};
 
-						ParseAddHelper(release, filter, trim, single, root, clean, out key);
+						ParseAddHelper(release, clean, out key);
 						break;
 					case ItemType.Rom:
 						Rom rom = new Rom()
@@ -1236,7 +1188,7 @@ namespace SabreTools.Helper.Dats
 							ItemStatus = status,
 						};
 
-						ParseAddHelper(rom, filter, trim, single, root, clean, out key);
+						ParseAddHelper(rom, clean, out key);
 						break;
 					case ItemType.Sample:
 						Sample sample = new Sample()
@@ -1250,7 +1202,7 @@ namespace SabreTools.Helper.Dats
 							},
 						};
 
-						ParseAddHelper(sample, filter, trim, single, root, clean, out key);
+						ParseAddHelper(sample, clean, out key);
 						break;
 				}
 			}			
@@ -1262,10 +1214,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
@@ -1279,14 +1227,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool keep,
 			bool clean,
@@ -1325,7 +1265,7 @@ namespace SabreTools.Helper.Dats
 							Rom rom = new Rom("null", tempgame);
 
 							// Now process and add the rom
-							ParseAddHelper(rom, filter, trim, single, root, clean, out key);
+							ParseAddHelper(rom, clean, out key);
 						}
 
 						// Regardless, end the current folder
@@ -1871,7 +1811,7 @@ namespace SabreTools.Helper.Dats
 										};
 
 										// Now process and add the rom
-										ParseAddHelper(olrom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(olrom, clean, out key);
 										break;
 
 									// For Software List and MAME listxml only
@@ -1973,7 +1913,7 @@ namespace SabreTools.Helper.Dats
 										};
 
 										// Now process and add the rom
-										ParseAddHelper(relrom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(relrom, clean, out key);
 
 										subreader.Read();
 										break;
@@ -2016,7 +1956,7 @@ namespace SabreTools.Helper.Dats
 										};
 
 										// Now process and add the rom
-										ParseAddHelper(biosrom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(biosrom, clean, out key);
 
 										subreader.Read();
 										break;
@@ -2044,7 +1984,7 @@ namespace SabreTools.Helper.Dats
 										};
 
 										// Now process and add the rom
-										ParseAddHelper(archiverom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(archiverom, clean, out key);
 
 										subreader.Read();
 										break;
@@ -2072,7 +2012,7 @@ namespace SabreTools.Helper.Dats
 										};
 
 										// Now process and add the rom
-										ParseAddHelper(samplerom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(samplerom, clean, out key);
 
 										subreader.Read();
 										break;
@@ -2215,7 +2155,7 @@ namespace SabreTools.Helper.Dats
 										}
 
 										// Now process and add the rom
-										ParseAddHelper(inrom, filter, trim, single, root, clean, out key);
+										ParseAddHelper(inrom, clean, out key);
 
 										subreader.Read();
 										break;
@@ -2390,7 +2330,7 @@ namespace SabreTools.Helper.Dats
 							}
 
 							// Now process and add the rom
-							ParseAddHelper(rom, filter, trim, single, root, clean, out key);
+							ParseAddHelper(rom, clean, out key);
 
 							xtr.Read();
 							break;
@@ -2417,10 +2357,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpMD5(
 			// Standard Dat parsing
@@ -2428,14 +2364,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2464,7 +2392,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2476,10 +2404,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpSFV(
 			// Standard Dat parsing
@@ -2487,14 +2411,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2523,7 +2439,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2535,10 +2451,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpSHA1(
 			// Standard Dat parsing
@@ -2546,14 +2458,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2582,7 +2486,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2594,10 +2498,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpSHA256(
 			// Standard Dat parsing
@@ -2605,14 +2505,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2641,7 +2533,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2653,10 +2545,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpSHA384(
 			// Standard Dat parsing
@@ -2664,14 +2552,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2700,7 +2580,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2712,10 +2592,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		private void ParseRedumpSHA512(
 			// Standard Dat parsing
@@ -2723,14 +2599,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean)
 		{
@@ -2759,7 +2627,7 @@ namespace SabreTools.Helper.Dats
 				};
 
 				// Now process and add the rom
-				ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+				ParseAddHelper(rom, clean, out string key);
 			}
 
 			sr.Dispose();
@@ -2771,10 +2639,6 @@ namespace SabreTools.Helper.Dats
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
 		/// <param name="srcid">Source ID for the DAT</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
 		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
 		/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
 		private void ParseRC(
@@ -2783,14 +2647,6 @@ namespace SabreTools.Helper.Dats
 			int sysid,
 			int srcid,
 
-			// Rom filtering
-			Filter filter,
-
-			// Rom renaming
-			bool trim,
-			bool single,
-			string root,
-
 			// Miscellaneous
 			bool clean,
 			bool descAsName)
@@ -2931,7 +2787,7 @@ namespace SabreTools.Helper.Dats
 						};
 
 						// Now process and add the rom
-						ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
+						ParseAddHelper(rom, clean, out string key);
 					}
 				}
 			}
@@ -2943,11 +2799,9 @@ namespace SabreTools.Helper.Dats
 		/// Add a rom to the Dat after checking
 		/// </summary>
 		/// <param name="item">Item data to check against</param>
-		/// <param name="filter">Filter object for passing to the DatItem level</param>
+		/// <param name="clean">True if the names should be cleaned to WoD standards, false otherwise</param>
-		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="key">Output param containing the key for the item</param>
-		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		private void ParseAddHelper(DatItem item, bool clean, out string key)
-		/// <param name="root">String representing root directory to compare against for length calculation</param>
-		private void ParseAddHelper(DatItem item, Filter filter, bool trim, bool single, string root, bool clean, out string key)
 		{
 			key = "";
 
@@ -3046,28 +2900,6 @@ namespace SabreTools.Helper.Dats
 				item = itemDisk;
 			}
 
-			// If the rom passes the filter, include it
-			if (filter.ItemPasses(item))
-			{
-				// If we are in single game mode, rename all games
-				if (single)
-				{
-					item.Machine.Name = "!";
-				}
-
-				// If we are in NTFS trim mode, trim the game name
-				if (trim)
-				{
-					// Windows max name length is 260
-					int usableLength = 260 - item.Machine.Name.Length - root.Length;
-					if (item.Name.Length > usableLength)
-					{
-						string ext = Path.GetExtension(item.Name);
-						item.Name = item.Name.Substring(0, usableLength - ext.Length);
-						item.Name += ext;
-					}
-				}
-
 			// Get the key and add statistical data
 			switch (item.Type)
 			{
@@ -3120,7 +2952,6 @@ namespace SabreTools.Helper.Dats
 			// Add the item to the DAT
 			Add(key, item);
 		}
-		}
 
 		#endregion
 	}

SabreTools.Helper/SabreTools.Helper.csproj

@@ -114,7 +114,7 @@
     <Compile Include="Data\Constants.cs" />
     <Compile Include="Data\Flags.cs" />
     <Compile Include="Data\Globals.cs" />
-    <Compile Include="Dats\Partials\DatFile.Bucketing.cs" />
+    <Compile Include="Dats\Partials\DatFile.Manipulate.cs" />
     <Compile Include="Dats\Partials\DatFile.ConvertUpdate.cs" />
     <Compile Include="Dats\Partials\DatFile.DFD.cs" />
     <Compile Include="Dats\Partials\DatFile.Parsers.cs" />

SabreTools/Partials/SabreTools.Inits.cs

@@ -325,7 +325,7 @@ namespace SabreTools
 			DatFile datdata = new DatFile();
 			foreach (string datfile in datfiles)
 			{
-				datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
+				datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
 			}
 			Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
@@ -361,7 +361,7 @@ namespace SabreTools
 			DatFile datdata = new DatFile();
 			foreach (string datfile in datfiles)
 			{
-				datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
+				datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
 			}
 			Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
@@ -728,7 +728,7 @@ namespace SabreTools
 			DatFile datdata = new DatFile();
 			foreach (string datfile in datfiles)
 			{
-				datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
+				datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
 			}
 			Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
@@ -756,7 +756,7 @@ namespace SabreTools
 			DatFile datdata = new DatFile();
 			foreach (string datfile in datfiles)
 			{
-				datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
+				datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
 			}
 			Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));