[ALL] File and namespace reorg

SabreTools.Library/DatFiles/DatFile.ConvertUpdate.cs

@@ -0,0 +1,636 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+using SabreTools.Library.Tools;
+
+#if MONO
+using System.IO;
+#else
+using Alphaleonis.Win32.Filesystem;
+
+using SearchOption = System.IO.SearchOption;
+#endif
+using NaturalSort;
+
+namespace SabreTools.Library.DatFiles
+{
+	public partial class DatFile
+	{
+		#region Converting and Updating
+
+		/// <summary>
+		/// Determine if input files should be merged, diffed, or processed invidually
+		/// </summary>
+		/// <param name="inputPaths">Names of the input files and/or folders</param>
+		/// <param name="basePaths">Names of base files and/or folders</param>
+		/// <param name="outDir">Optional param for output directory</param>
+		/// <param name="merge">True if input files should be merged into a single file, false otherwise</param>
+		/// <param name="diff">Non-zero flag for diffing mode, zero otherwise</param>
+		/// <param name="inplace">True if the output files should overwrite their inputs, false otherwise</param>
+		/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
+		/// <param name="bare">True if the date should not be appended to the default name, false otherwise [OBSOLETE]</param>
+		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
+		/// <param name="filter">Filter object to be passed to the DatItem level</param>
+		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
+		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		/// <param name="root">String representing root directory to compare against for length calculation</param>
+		public void DetermineUpdateType(List<string> inputPaths, List<string> basePaths, string outDir, bool merge, DiffMode diff, bool inplace, bool skip,
+			bool bare, bool clean, bool remUnicode, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root)
+		{
+			// If we're in merging or diffing mode, use the full list of inputs
+			if (merge || (diff != 0 && (diff & DiffMode.Against) == 0))
+			{
+				// Make sure there are no folders in inputs
+				List<string> newInputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true);
+
+				// Reverse if we have to
+				if ((diff & DiffMode.ReverseCascade) != 0)
+				{
+					newInputFileNames.Reverse();
+				}
+
+				// Create a dictionary of all ROMs from the input DATs
+				List<DatFile> datHeaders = PopulateUserData(newInputFileNames, inplace, clean,
+					remUnicode, descAsName, outDir, filter, splitType, trim, single, root);
+
+				// Modify the Dictionary if necessary and output the results
+				if (diff != 0 && diff < DiffMode.Cascade)
+				{
+					DiffNoCascade(diff, outDir, newInputFileNames);
+				}
+				// If we're in cascade and diff, output only cascaded diffs
+				else if (diff != 0 && diff >= DiffMode.Cascade)
+				{
+					DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip);
+				}
+				// Output all entries with user-defined merge
+				else
+				{
+					MergeNoDiff(outDir, newInputFileNames, datHeaders);
+				}
+			}
+			// If we're in "diff against" mode, we treat the inputs differently
+			else if ((diff & DiffMode.Against) != 0)
+			{
+				DiffAgainst(inputPaths, basePaths, outDir, inplace, clean, remUnicode, descAsName, filter, splitType, trim, single, root);
+			}
+			// Otherwise, loop through all of the inputs individually
+			else
+			{
+				Update(inputPaths, outDir, inplace, clean, remUnicode, descAsName, filter, splitType, trim, single, root);
+			}
+			return;
+		}
+
+		/// <summary>
+		/// Populate the user DatData object from the input files
+		/// </summary>
+		/// <param name="inputs">Paths to DATs to parse</param>
+		/// <param name="inplace">True if the output files should overwrite their inputs, false otherwise</param>
+		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
+		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
+		/// <param name="outDir">Optional param for output directory</param>
+		/// <param name="filter">Filter object to be passed to the DatItem level</param>
+		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
+		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		/// <param name="root">String representing root directory to compare against for length calculation</param>
+		/// <returns>List of DatData objects representing headers</returns>
+		private List<DatFile> PopulateUserData(List<string> inputs, bool inplace, bool clean, bool remUnicode, bool descAsName,
+			string outDir, Filter filter, SplitType splitType, bool trim, bool single, string root)
+		{
+			DatFile[] datHeaders = new DatFile[inputs.Count];
+			InternalStopwatch watch = new InternalStopwatch("Processing individual DATs");
+
+			// Parse all of the DATs into their own DatFiles in the array
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
+			{
+				string input = inputs[i];
+				Globals.Logger.User("Adding DAT: {0}", input.Split('¬')[0]);
+				datHeaders[i] = new DatFile
+				{
+					DatFormat = (DatFormat != 0 ? DatFormat : 0),
+					DedupeRoms = DedupeRoms,
+				};
+
+				datHeaders[i].Parse(input.Split('¬')[0], i, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
+			});
+
+			watch.Stop();
+
+			watch.Start("Populating internal DAT");
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
+			{
+				// Get the list of keys from the DAT
+				List<string> keys = datHeaders[i].Keys.ToList();
+				foreach (string key in keys)
+				{
+					// Add everything from the key to the internal DAT
+					AddRange(key, datHeaders[i][key]);
+
+					// Now remove the key from the source DAT
+					datHeaders[i].Remove(key);
+				}
+
+				// Now remove the file dictionary from the souce DAT to save memory
+				datHeaders[i].DeleteDictionary();
+			});
+
+			// Now that we have a merged DAT, filter it
+			Filter(filter, single, trim, root);
+
+			watch.Stop();
+
+			return datHeaders.ToList();
+		}
+
+		/// <summary>
+		/// Output diffs against a base set
+		/// </summary>
+		/// <param name="inputPaths">Names of the input files and/or folders</param>
+		/// <param name="basePaths">Names of base files and/or folders</param>
+		/// <param name="outDir">Optional param for output directory</param>
+		/// <param name="inplace">True if the output files should overwrite their inputs, false otherwise</param>
+		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
+		/// <param name="filter">Filter object to be passed to the DatItem level</param>
+		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
+		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		/// <param name="root">String representing root directory to compare against for length calculation</param>
+		public void DiffAgainst(List<string> inputPaths, List<string> basePaths, string outDir, bool inplace, bool clean, bool remUnicode,
+			bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root)
+		{
+			// First we want to parse all of the base DATs into the input
+			InternalStopwatch watch = new InternalStopwatch("Populating base DAT for comparison...");
+
+			List<string> baseFileNames = FileTools.GetOnlyFilesFromInputs(basePaths);
+			Parallel.ForEach(baseFileNames, Globals.ParallelOptions, path =>
+			{
+				Parse(path, 0, 0, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
+			});
+
+			watch.Stop();
+
+			// For comparison's sake, we want to use CRC as the base ordering
+			BucketBy(SortedBy.CRC, DedupeType.Full);
+
+			// Now we want to compare each input DAT against the base
+			List<string> inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true);
+			foreach (string path in inputFileNames)
+			{
+				// Get the two halves of the path
+				string[] splitpath = path.Split('¬');
+
+				Globals.Logger.User("Comparing '{0}'' to base DAT", splitpath[0]);
+
+				// First we parse in the DAT internally
+				DatFile intDat = new DatFile();
+				intDat.Parse(splitpath[0], 1, 1, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
+
+				// For comparison's sake, we want to use CRC as the base ordering
+				intDat.BucketBy(SortedBy.CRC, DedupeType.Full);
+
+				// Then we do a hashwise comparison against the base DAT
+				List<string> keys = intDat.Keys.ToList();
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+				{
+					List<DatItem> datItems = intDat[key];
+					List<DatItem> keepDatItems = new List<DatItem>();
+					foreach (DatItem datItem in datItems)
+					{
+						if (!datItem.HasDuplicates(this, true))
+						{
+							keepDatItems.Add(datItem);
+						}
+					}
+
+					// Now add the new list to the key
+					intDat.Remove(key);
+					intDat.AddRange(key, keepDatItems);
+				});
+
+				// Determine the output path for the DAT
+				string interOutDir = outDir;
+				if (inplace)
+				{
+					interOutDir = Path.GetDirectoryName(path);
+				}
+				else if (!String.IsNullOrEmpty(interOutDir))
+				{
+					interOutDir = Path.GetDirectoryName(Path.Combine(interOutDir, splitpath[0].Remove(0, splitpath[1].Length + 1)));
+				}
+				else
+				{
+					interOutDir = Path.GetDirectoryName(Path.Combine(Environment.CurrentDirectory, splitpath[0].Remove(0, splitpath[1].Length + 1)));
+				}
+
+				// Once we're done, try writing out
+				intDat.WriteToFile(interOutDir);
+
+				// Due to possible memory requirements, we force a garbage collection
+				GC.Collect();
+			}
+		}
+
+		/// <summary>
+		/// Output cascading diffs
+		/// </summary>
+		/// <param name="outDir">Output directory to write the DATs to</param>
+		/// <param name="inplace">True if cascaded diffs are outputted in-place, false otherwise</param>
+		/// <param name="inputs">List of inputs to write out from</param>
+		/// <param name="datHeaders">Dat headers used optionally</param>
+		/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
+		public void DiffCascade(string outDir, bool inplace, List<string> inputs, List<DatFile> datHeaders, bool skip)
+		{
+			string post = "";
+
+			// Create a list of DatData objects representing output files
+			List<DatFile> outDats = new List<DatFile>();
+
+			// Loop through each of the inputs and get or create a new DatData object
+			InternalStopwatch watch = new InternalStopwatch("Initializing all output DATs");
+
+			DatFile[] outDatsArray = new DatFile[inputs.Count];
+
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
+			{
+				string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
+				DatFile diffData;
+
+				// If we're in inplace mode, take the appropriate DatData object already stored
+				if (inplace || outDir != Environment.CurrentDirectory)
+				{
+					diffData = datHeaders[j];
+				}
+				else
+				{
+					diffData = new DatFile(this);
+					diffData.FileName += post;
+					diffData.Name += post;
+					diffData.Description += post;
+				}
+				diffData.ResetDictionary();
+
+				outDatsArray[j] = diffData;
+			});
+
+			outDats = outDatsArray.ToList();
+			watch.Stop();
+
+			// Now, loop through the dictionary and populate the correct DATs
+			watch.Start("Populating all output DATs");
+			List<string> keys = Keys.ToList();
+
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = DatItem.Merge(this[key]);
+
+				// If the rom list is empty or null, just skip it
+				if (items == null || items.Count == 0)
+				{
+					return;
+				}
+
+				foreach (DatItem item in items)
+				{
+					// There's odd cases where there are items with System ID < 0. Skip them for now
+					if (item.SystemID < 0)
+					{
+						Globals.Logger.Warning("Item found with a <0 SystemID: {0}", item.Name);
+						continue;
+					}
+
+					outDats[item.SystemID].Add(key, item);
+				}
+			});
+			
+			watch.Stop();
+
+			// Finally, loop through and output each of the DATs
+			watch.Start("Outputting all created DATs");
+
+			Parallel.For((skip ? 1 : 0), inputs.Count, Globals.ParallelOptions, j =>
+			{
+				// If we have an output directory set, replace the path
+				string path = "";
+				if (inplace)
+				{
+					path = Path.GetDirectoryName(inputs[j].Split('¬')[0]);
+				}
+				else if (outDir != Environment.CurrentDirectory)
+				{
+					string[] split = inputs[j].Split('¬');
+					path = outDir + (split[0] == split[1]
+						? Path.GetFileName(split[0])
+						: (Path.GetDirectoryName(split[0]).Remove(0, split[1].Length))); ;
+				}
+
+				// Try to output the file
+				outDats[j].WriteToFile(path);
+			});
+
+			watch.Stop();
+		}
+
+		/// <summary>
+		/// Output non-cascading diffs
+		/// </summary>
+		/// <param name="diff">Non-zero flag for diffing mode, zero otherwise</param>
+		/// <param name="outDir">Output directory to write the DATs to</param>
+		/// <param name="inputs">List of inputs to write out from</param>
+		public void DiffNoCascade(DiffMode diff, string outDir, List<string> inputs)
+		{
+			InternalStopwatch watch = new InternalStopwatch("Initializing all output DATs");
+
+			// Default vars for use
+			string post = "";
+			DatFile outerDiffData = new DatFile();
+			DatFile dupeData = new DatFile();
+
+			// Fill in any information not in the base DAT
+			if (String.IsNullOrEmpty(FileName))
+			{
+				FileName = "All DATs";
+			}
+			if (String.IsNullOrEmpty(Name))
+			{
+				Name = "All DATs";
+			}
+			if (String.IsNullOrEmpty(Description))
+			{
+				Description = "All DATs";
+			}
+
+			// Don't have External dupes
+			if ((diff & DiffMode.NoDupes) != 0)
+			{
+				post = " (No Duplicates)";
+				outerDiffData = new DatFile(this);
+				outerDiffData.FileName += post;
+				outerDiffData.Name += post;
+				outerDiffData.Description += post;
+				outerDiffData.ResetDictionary();
+			}
+
+			// Have External dupes
+			if ((diff & DiffMode.Dupes) != 0)
+			{
+				post = " (Duplicates)";
+				dupeData = new DatFile(this);
+				dupeData.FileName += post;
+				dupeData.Name += post;
+				dupeData.Description += post;
+				dupeData.ResetDictionary();
+			}
+
+			// Create a list of DatData objects representing individual output files
+			List<DatFile> outDats = new List<DatFile>();
+
+			// Loop through each of the inputs and get or create a new DatData object
+			if ((diff & DiffMode.Individuals) != 0)
+			{
+				DatFile[] outDatsArray = new DatFile[inputs.Count];
+
+				Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
+				{
+					string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
+					DatFile diffData = new DatFile(this);
+					diffData.FileName += innerpost;
+					diffData.Name += innerpost;
+					diffData.Description += innerpost;
+					diffData.ResetDictionary();
+					outDatsArray[j] = diffData;
+				});
+
+				outDats = outDatsArray.ToList();
+			}
+
+			watch.Stop();
+
+			// Now, loop through the dictionary and populate the correct DATs
+			watch.Start("Populating all output DATs");
+
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = DatItem.Merge(this[key]);
+
+				// If the rom list is empty or null, just skip it
+				if (items == null || items.Count == 0)
+				{
+					return;
+				}
+
+				// Loop through and add the items correctly
+				foreach(DatItem item in items)
+				{
+					// No duplicates
+					if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
+					{
+						if ((item.Dupe & DupeType.Internal) != 0)
+						{
+							// Individual DATs that are output
+							if ((diff & DiffMode.Individuals) != 0)
+							{
+								outDats[item.SystemID].Add(key, item);
+							}
+
+							// Merged no-duplicates DAT
+							if ((diff & DiffMode.NoDupes) != 0)
+							{
+								DatItem newrom = item.Clone() as DatItem;
+								newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
+
+								outerDiffData.Add(key, newrom);
+							}
+						}
+					}
+
+					// Duplicates only
+					if ((diff & DiffMode.Dupes) != 0)
+					{
+						if ((item.Dupe & DupeType.External) != 0)
+						{
+							DatItem newrom = item.Clone() as DatItem;
+							newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
+
+							dupeData.Add(key, newrom);
+						}
+					}
+				}
+			});
+
+			watch.Stop();
+
+			// Finally, loop through and output each of the DATs
+			watch.Start("Outputting all created DATs");
+
+			// Output the difflist (a-b)+(b-a) diff
+			if ((diff & DiffMode.NoDupes) != 0)
+			{
+				outerDiffData.WriteToFile(outDir);
+			}
+
+			// Output the (ab) diff
+			if ((diff & DiffMode.Dupes) != 0)
+			{
+				dupeData.WriteToFile(outDir);
+			}
+
+			// Output the individual (a-b) DATs
+			if ((diff & DiffMode.Individuals) != 0)
+			{
+				Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
+				{
+					// If we have an output directory set, replace the path
+					string[] split = inputs[j].Split('¬');
+					string path = Path.Combine(outDir,
+						(split[0] == split[1]
+							? Path.GetFileName(split[0])
+							: (Path.GetDirectoryName(split[0]).Remove(0, split[1].Length))));
+
+					// Try to output the file
+					outDats[j].WriteToFile(path);
+				});
+			}
+
+			watch.Stop();
+		}
+
+		/// <summary>
+		/// Output user defined merge
+		/// </summary>
+		/// <param name="outDir">Output directory to write the DATs to</param>
+		/// <param name="inputs">List of inputs to write out from</param>
+		/// <param name="datHeaders">Dat headers used optionally</param>
+		public void MergeNoDiff(string outDir, List<string> inputs, List<DatFile> datHeaders)
+		{
+			// If we're in SuperDAT mode, prefix all games with their respective DATs
+			if (Type == "SuperDAT")
+			{
+				List<string> keys = Keys.ToList();
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+				{
+					List<DatItem> items = this[key].ToList();
+					List<DatItem> newItems = new List<DatItem>();
+					foreach (DatItem item in items)
+					{
+						DatItem newItem = item;
+						string filename = inputs[newItem.SystemID].Split('¬')[0];
+						string rootpath = inputs[newItem.SystemID].Split('¬')[1];
+
+						rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString());
+						filename = filename.Remove(0, rootpath.Length);
+						newItem.MachineName = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+							+ Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+							+ newItem.MachineName;
+
+						newItems.Add(newItem);
+					}
+
+					Remove(key);
+					AddRange(key, newItems);
+				});
+			}
+
+			// Try to output the file
+			WriteToFile(outDir);
+		}
+
+		/// <summary>
+		/// Convert, update, and filter a DAT file or set of files using a base
+		/// </summary>
+		/// <param name="inputFileNames">Names of the input files and/or folders</param>
+		/// <param name="outDir">Optional param for output directory</param>
+		/// <param name="merge">True if input files should be merged into a single file, false otherwise</param>
+		/// <param name="diff">Non-zero flag for diffing mode, zero otherwise</param>
+		/// <param name="inplace">True if the output files should overwrite their inputs, false otherwise</param>
+		/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
+		/// <param name="bare">True if the date should not be appended to the default name, false otherwise [OBSOLETE]</param>
+		/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		/// <param name="descAsName">True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)</param>
+		/// <param name="filter">Filter object to be passed to the DatItem level</param>
+		/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
+		/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
+		/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
+		/// <param name="root">String representing root directory to compare against for length calculation</param>
+		public void Update(List<string> inputFileNames, string outDir, bool inplace, bool clean, bool remUnicode, bool descAsName,
+			Filter filter, SplitType splitType, bool trim, bool single, string root)
+		{
+			for (int i = 0; i < inputFileNames.Count; i++)
+			{
+				// Get the input file name
+				string inputFileName = inputFileNames[i];
+
+				// Clean the input string
+				if (inputFileName != "")
+				{
+					inputFileName = Path.GetFullPath(inputFileName);
+				}
+
+				if (File.Exists(inputFileName))
+				{
+					// If inplace is set, override the output dir
+					string realOutDir = outDir;
+					if (inplace)
+					{
+						realOutDir = Path.GetDirectoryName(inputFileName);
+					}
+
+					DatFile innerDatdata = new DatFile(this);
+					Globals.Logger.User("Processing '{0}'", Path.GetFileName(inputFileName));
+					innerDatdata.Parse(inputFileName, 0, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName,
+						keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
+					innerDatdata.Filter(filter, trim, single, root);
+
+					// Try to output the file
+					innerDatdata.WriteToFile((realOutDir == Environment.CurrentDirectory ? Path.GetDirectoryName(inputFileName) : realOutDir), overwrite: (realOutDir != Environment.CurrentDirectory));
+				}
+				else if (Directory.Exists(inputFileName))
+				{
+					inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
+
+					// If inplace is set, override the output dir
+					string realOutDir = outDir;
+					if (inplace)
+					{
+						realOutDir = Path.GetDirectoryName(inputFileName);
+					}
+
+					List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList();
+					Parallel.ForEach(subFiles, Globals.ParallelOptions, file =>
+					{
+						Globals.Logger.User("Processing '{0}'", Path.GetFullPath(file).Remove(0, inputFileName.Length));
+						DatFile innerDatdata = new DatFile(this);
+						innerDatdata.Parse(file, 0, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName,
+							keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
+						innerDatdata.Filter(filter, trim, single, root);
+
+						// Try to output the file
+						innerDatdata.WriteToFile((realOutDir == Environment.CurrentDirectory ? Path.GetDirectoryName(file) : realOutDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)),
+							overwrite: (realOutDir != Environment.CurrentDirectory));
+					});
+				}
+				else
+				{
+					Globals.Logger.Error("I'm sorry but '{0}' doesn't exist!", inputFileName);
+					return;
+				}
+			}
+		}
+
+		#endregion
+	}
+}

SabreTools.Library/DatFiles/DatFile.DFD.cs

@@ -0,0 +1,423 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+using SabreTools.Library.Tools;
+
+#if MONO
+using System.IO;
+#else
+using Alphaleonis.Win32.Filesystem;
+
+using IOException = System.IO.IOException;
+using SearchOption = System.IO.SearchOption;
+#endif
+using SharpCompress.Common;
+
+namespace SabreTools.Library.DatFiles
+{
+	public partial class DatFile
+	{
+		#region Populate DAT from Directory [MODULAR DONE, FOR NOW]
+
+		/// <summary>
+		/// Create a new Dat from a directory
+		/// </summary>
+		/// <param name="basePath">Base folder to be used in creating the DAT</param>
+		/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated</param>
+		/// <param name="bare">True if the date should be omitted from the DAT, false otherwise</param>
+		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
+		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
+		/// <param name="skipFileType">Type of files that should be skipped</param>
+		/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
+		/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
+		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
+		/// <param name="outDir">Output directory to </param>
+		/// <param name="copyFiles">True if files should be copied to the temp directory before hashing, false otherwise</param>
+		/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
+		public bool PopulateFromDir(string basePath, Hash omitFromScan, bool bare, bool archivesAsFiles, bool enableGzip,
+			 SkipFileType skipFileType, bool addBlanks, bool addDate, string tempDir, bool copyFiles, string headerToCheckAgainst)
+		{
+			// If the description is defined but not the name, set the name from the description
+			if (String.IsNullOrEmpty(Name) && !String.IsNullOrEmpty(Description))
+			{
+				Name = Description;
+			}
+
+			// If the name is defined but not the description, set the description from the name
+			else if (!String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
+			{
+				Description = Name + (bare ? "" : " (" + Date + ")");
+			}
+
+			// If neither the name or description are defined, set them from the automatic values
+			else if (String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
+			{
+				Name = basePath.Split(Path.DirectorySeparatorChar).Last();
+				Description = Name + (bare ? "" : " (" + Date + ")");
+			}
+
+			// Process the input
+			if (Directory.Exists(basePath))
+			{
+				Globals.Logger.Verbose("Folder found: {0}", basePath);
+
+				// Process the files in the main folder
+				List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
+				Parallel.ForEach(files, Globals.ParallelOptions, item =>
+				{
+					PopulateFromDirCheckFile(item, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
+						addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
+				});
+
+				// Find all top-level subfolders
+				files = Directory.EnumerateDirectories(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
+				foreach (string item in files)
+				{
+					List<string> subfiles = Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories).ToList();
+					Parallel.ForEach(subfiles, Globals.ParallelOptions, subitem =>
+					{
+						PopulateFromDirCheckFile(subitem, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
+							addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
+					});
+				}
+
+				// Now find all folders that are empty, if we are supposed to
+				if (!Romba && addBlanks)
+				{
+					List<string> empties = FileTools.GetEmptyDirectories(basePath).ToList();
+					Parallel.ForEach(empties, Globals.ParallelOptions, dir =>
+					{
+						// Get the full path for the directory
+						string fulldir = Path.GetFullPath(dir);
+
+						// Set the temporary variables
+						string gamename = "";
+						string romname = "";
+
+						// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
+						if (Type == "SuperDAT")
+						{
+							gamename = fulldir.Remove(0, basePath.Length + 1);
+							romname = "_";
+						}
+
+						// Otherwise, we want just the top level folder as the game, and the file as everything else
+						else
+						{
+							gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0];
+							romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "_");
+						}
+
+						// Sanitize the names
+						if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
+						{
+							gamename = gamename.Substring(1);
+						}
+						if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
+						{
+							gamename = gamename.Substring(0, gamename.Length - 1);
+						}
+						if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
+						{
+							romname = romname.Substring(1);
+						}
+						if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
+						{
+							romname = romname.Substring(0, romname.Length - 1);
+						}
+
+						Globals.Logger.Verbose("Adding blank empty folder: {0}", gamename);
+						this["null"].Add(new Rom(romname, gamename, omitFromScan));
+					});
+				}
+			}
+			else if (File.Exists(basePath))
+			{
+				PopulateFromDirCheckFile(basePath, Path.GetDirectoryName(Path.GetDirectoryName(basePath)), omitFromScan, bare, archivesAsFiles, enableGzip,
+					skipFileType, addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
+			}
+
+			// Now that we're done, delete the temp folder (if it's not the default)
+			Globals.Logger.User("Cleaning temp folder");
+			if (tempDir != Path.GetTempPath())
+			{
+				FileTools.TryDeleteDirectory(tempDir);
+			}
+
+			return true;
+		}
+
+		/// <summary>
+		/// Check a given file for hashes, based on current settings
+		/// </summary>
+		/// <param name="item">Filename of the item to be checked</param>
+		/// <param name="basePath">Base folder to be used in creating the DAT</param>
+		/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated</param>
+		/// <param name="bare">True if the date should be omitted from the DAT, false otherwise</param>
+		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
+		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
+		/// <param name="skipFileType">Type of files that should be skipped</param>
+		/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
+		/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
+		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
+		/// <param name="copyFiles">True if files should be copied to the temp directory before hashing, false otherwise</param>
+		/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
+		private void PopulateFromDirCheckFile(string item, string basePath, Hash omitFromScan, bool bare, bool archivesAsFiles,
+			bool enableGzip, SkipFileType skipFileType, bool addBlanks, bool addDate, string tempDir, bool copyFiles, string headerToCheckAgainst)
+		{
+			// Define the temporary directory
+			string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar;
+
+			// Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
+			if (Romba)
+			{
+				Rom rom = ArchiveTools.GetTorrentGZFileInfo(item);
+
+				// If the rom is valid, write it out
+				if (rom != null && rom.Name != null)
+				{
+					// Add the list if it doesn't exist already
+					Add(rom.Size + "-" + rom.CRC, rom);
+					Globals.Logger.User("File added: {0}", Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+				}
+				else
+				{
+					Globals.Logger.User("File not added: {0}", Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+					return;
+				}
+
+				return;
+			}
+
+			// If we're copying files, copy it first and get the new filename
+			string newItem = item;
+			string newBasePath = basePath;
+			if (copyFiles)
+			{
+				newBasePath = Path.Combine(tempDir, Path.GetRandomFileName());
+				newItem = Path.GetFullPath(Path.Combine(newBasePath, Path.GetFullPath(item).Remove(0, basePath.Length + 1)));
+				Directory.CreateDirectory(Path.GetDirectoryName(newItem));
+				File.Copy(item, newItem, true);
+			}
+
+			// Create a list for all found items
+			List<Rom> extracted = null;
+
+			// Temporarily set the archivesAsFiles if we have a GZip archive and we're not supposed to use it as one
+			if (archivesAsFiles && !enableGzip && newItem.EndsWith(".gz"))
+			{
+				archivesAsFiles = false;
+			}
+			
+			// If we don't have archives as files, try to scan the file as an archive
+			if (!archivesAsFiles)
+			{
+				// If all deep hash skip flags are set, do a quickscan
+				if (omitFromScan == Hash.SecureHashes)
+				{
+					extracted = ArchiveTools.GetArchiveFileInfo(newItem, date: addDate);
+				}
+				// Otherwise, get the list with whatever hashes are wanted
+				else
+				{
+					extracted = ArchiveTools.GetExtendedArchiveFileInfo(newItem, omitFromScan: omitFromScan, date: addDate);
+				}
+			}
+
+			// If the file should be skipped based on type, do so now
+			if ((extracted != null && skipFileType == SkipFileType.Archive)
+				|| (extracted == null && skipFileType == SkipFileType.File))
+			{
+				return;
+			}
+
+			// If the extracted list is null, just scan the item itself
+			if (extracted == null || archivesAsFiles)
+			{
+				PopulateFromDirProcessFile(newItem, "", newBasePath, omitFromScan, addDate, headerToCheckAgainst);
+			}
+			// Otherwise, add all of the found items
+			else
+			{
+				// First take care of the found items
+				Parallel.ForEach(extracted, Globals.ParallelOptions, rom =>
+				{
+					PopulateFromDirProcessFileHelper(newItem,
+						rom,
+						basePath,
+						(Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item));
+				});
+
+				// Then, if we're looking for blanks, get all of the blank folders and add them
+				if (addBlanks)
+				{
+					List<string> empties = ArchiveTools.GetEmptyFoldersInArchive(newItem);
+					Parallel.ForEach(empties, Globals.ParallelOptions, empty =>
+					{
+						Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan);
+						PopulateFromDirProcessFileHelper(newItem,
+							emptyRom,
+							basePath,
+							(Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item));
+					});
+				}
+			}
+
+			// Cue to delete the file if it's a copy
+			if (copyFiles && item != newItem)
+			{
+				FileTools.TryDeleteDirectory(newBasePath);
+			}
+
+			// Delete the sub temp directory
+			FileTools.TryDeleteDirectory(tempSubDir);
+		}
+
+		/// <summary>
+		/// Process a single file as a file
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="parent">Parent game to be used</param>
+		/// <param name="basePath">Path the represents the parent directory</param>
+		/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated</param>
+		/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
+		/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
+		private void PopulateFromDirProcessFile(string item, string parent, string basePath, Hash omitFromScan,
+			bool addDate, string headerToCheckAgainst)
+		{
+			Globals.Logger.Verbose("'{0}' treated like a file", Path.GetFileName(item));
+			Rom rom = FileTools.GetFileInfo(item, omitFromScan: omitFromScan, date: addDate, header: headerToCheckAgainst);
+
+			PopulateFromDirProcessFileHelper(item, rom, basePath, parent);
+		}
+
+		/// <summary>
+		/// Process a single file as a file (with found Rom data)
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="item">Rom data to be used to write to file</param>
+		/// <param name="basepath">Path the represents the parent directory</param>
+		/// <param name="parent">Parent game to be used</param>
+		private void PopulateFromDirProcessFileHelper(string item, DatItem datItem, string basepath, string parent)
+		{
+			// If the datItem isn't a Rom or Disk, return
+			if (datItem.Type != ItemType.Rom && datItem.Type != ItemType.Disk)
+			{
+				return;
+			}
+
+			string key = "";
+			if (datItem.Type == ItemType.Rom)
+			{
+				key = ((Rom)datItem).Size + "-" + ((Rom)datItem).CRC;
+			}
+			else
+			{
+				key = ((Disk)datItem).MD5;
+			}
+
+			// Add the list if it doesn't exist already
+			Add(key);
+
+			try
+			{
+				// If the basepath ends with a directory separator, remove it
+				if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					basepath += Path.DirectorySeparatorChar.ToString();
+				}
+
+				// Make sure we have the full item path
+				item = Path.GetFullPath(item);
+
+				// Get the data to be added as game and item names
+				string gamename = "";
+				string romname = "";
+
+				// If the parent is blank, then we have a non-archive file
+				if (parent == "")
+				{
+					// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
+					if (Type == "SuperDAT")
+					{
+						gamename = Path.GetDirectoryName(item.Remove(0, basepath.Length));
+						romname = Path.GetFileName(item);
+					}
+
+					// Otherwise, we want just the top level folder as the game, and the file as everything else
+					else
+					{
+						gamename = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0];
+						romname = item.Remove(0, (Path.Combine(basepath, gamename).Length));
+					}
+				}
+
+				// Otherwise, we assume that we have an archive
+				else
+				{
+					// If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
+					if (Type == "SuperDAT")
+					{
+						gamename = parent;
+						romname = datItem.Name;
+					}
+
+					// Otherwise, we want the archive name as the game, and the file as everything else
+					else
+					{
+						gamename = parent;
+						romname = datItem.Name;
+					}
+				}
+
+				// Sanitize the names
+				if (romname == null)
+				{
+					romname = "";
+				}
+				if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					gamename = gamename.Substring(1);
+				}
+				if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					gamename = gamename.Substring(0, gamename.Length - 1);
+				}
+				if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					romname = romname.Substring(1);
+				}
+				if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					romname = romname.Substring(0, romname.Length - 1);
+				}
+				if (!String.IsNullOrEmpty(gamename) && String.IsNullOrEmpty(romname))
+				{
+					romname = gamename;
+					gamename = "Default";
+				}
+
+				// Update rom information
+				datItem.Name = romname;
+				datItem.MachineName = gamename;
+				datItem.Description = gamename;
+
+				// Add the file information to the DAT
+				Add(key, datItem);
+
+				Globals.Logger.User("File added: {0}", romname + Environment.NewLine);
+			}
+			catch (IOException ex)
+			{
+				Globals.Logger.Error(ex.ToString());
+				return;
+			}
+		}
+
+		#endregion
+	}
+}

SabreTools.Library/DatFiles/DatFile.Splitters.cs

@@ -0,0 +1,633 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Web;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+using SabreTools.Library.Tools;
+
+#if MONO
+using System.IO;
+#else
+using Alphaleonis.Win32.Filesystem;
+#endif
+using NaturalSort;
+
+namespace SabreTools.Library.DatFiles
+{
+	public partial class DatFile
+	{
+		#region Splitting
+
+		/// <summary>
+		/// Split a DAT by input extensions
+		/// </summary>
+		/// <param name="outDir">Name of the directory to write the DATs out to</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <param name="extA">List of extensions to split on (first DAT)</param>
+		/// <param name="extB">List of extensions to split on (second DAT)</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		public bool SplitByExt(string outDir, string basepath, List<string> extA, List<string> extB)
+		{
+			// Make sure all of the extensions have a dot at the beginning
+			List<string> newExtA = new List<string>();
+			foreach (string s in extA)
+			{
+				newExtA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
+			}
+			string newExtAString = string.Join(",", newExtA);
+
+			List<string> newExtB = new List<string>();
+			foreach (string s in extB)
+			{
+				newExtB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
+			}
+			string newExtBString = string.Join(",", newExtB);
+
+			// Set all of the appropriate outputs for each of the subsets
+			DatFile datdataA = new DatFile
+			{
+				FileName = this.FileName + " (" + newExtAString + ")",
+				Name = this.Name + " (" + newExtAString + ")",
+				Description = this.Description + " (" + newExtAString + ")",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				DatFormat = this.DatFormat,
+			};
+			DatFile datdataB = new DatFile
+			{
+				FileName = this.FileName + " (" + newExtBString + ")",
+				Name = this.Name + " (" + newExtBString + ")",
+				Description = this.Description + " (" + newExtBString + ")",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				DatFormat = this.DatFormat,
+			};
+
+			// If roms is empty, return false
+			if (Count == 0)
+			{
+				return false;
+			}
+
+			// Now separate the roms accordingly
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = this[key];
+				foreach (DatItem item in items)
+				{
+					if (newExtA.Contains(Path.GetExtension(item.Name.ToUpperInvariant())))
+					{
+						datdataA.Add(key, item);
+					}
+					else if (newExtB.Contains(Path.GetExtension(item.Name.ToUpperInvariant())))
+					{
+						datdataB.Add(key, item);
+					}
+					else
+					{
+						datdataA.Add(key, item);
+						datdataB.Add(key, item);
+					}
+				}
+			});
+
+			// Get the output directory
+			if (outDir != "")
+			{
+				outDir = outDir + Path.GetDirectoryName(this.FileName).Remove(0, basepath.Length - 1);
+			}
+			else
+			{
+				outDir = Path.GetDirectoryName(this.FileName);
+			}
+
+			// Then write out both files
+			bool success = datdataA.WriteToFile(outDir);
+			success &= datdataB.WriteToFile(outDir);
+
+			return success;
+		}
+
+		/// <summary>
+		/// Split a DAT by best available hashes
+		/// </summary>
+		/// <param name="outDir">Name of the directory to write the DATs out to</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		public bool SplitByHash(string outDir, string basepath)
+		{
+			// Sanitize the basepath to be more predictable
+			basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
+
+			// Create each of the respective output DATs
+			Globals.Logger.User("Creating and populating new DATs");
+			DatFile nodump = new DatFile
+			{
+				FileName = this.FileName + " (Nodump)",
+				Name = this.Name + " (Nodump)",
+				Description = this.Description + " (Nodump)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile sha512 = new DatFile
+			{
+				FileName = this.FileName + " (SHA-512)",
+				Name = this.Name + " (SHA-512)",
+				Description = this.Description + " (SHA-512)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile sha384 = new DatFile
+			{
+				FileName = this.FileName + " (SHA-384)",
+				Name = this.Name + " (SHA-384)",
+				Description = this.Description + " (SHA-384)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile sha256 = new DatFile
+			{
+				FileName = this.FileName + " (SHA-256)",
+				Name = this.Name + " (SHA-256)",
+				Description = this.Description + " (SHA-256)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile sha1 = new DatFile
+			{
+				FileName = this.FileName + " (SHA-1)",
+				Name = this.Name + " (SHA-1)",
+				Description = this.Description + " (SHA-1)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile md5 = new DatFile
+			{
+				FileName = this.FileName + " (MD5)",
+				Name = this.Name + " (MD5)",
+				Description = this.Description + " (MD5)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile crc = new DatFile
+			{
+				FileName = this.FileName + " (CRC)",
+				Name = this.Name + " (CRC)",
+				Description = this.Description + " (CRC)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile other = new DatFile
+			{
+				FileName = this.FileName + " (Other)",
+				Name = this.Name + " (Other)",
+				Description = this.Description + " (Other)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+
+			// Now populate each of the DAT objects in turn
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = this[key];
+				foreach (DatItem item in items)
+				{
+					// If the file is not a Rom or Disk, continue
+					if (item.Type != ItemType.Disk && item.Type != ItemType.Rom)
+					{
+						return;
+					}
+
+					// If the file is a nodump
+					if ((item.Type == ItemType.Rom && ((Rom)item).ItemStatus == ItemStatus.Nodump)
+						|| (item.Type == ItemType.Disk && ((Disk)item).ItemStatus == ItemStatus.Nodump))
+					{
+						nodump.Add(key, item);
+					}
+					// If the file has a SHA-512
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA512))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA512)))
+					{
+						sha512.Add(key, item);
+					}
+					// If the file has a SHA-384
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA384))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA384)))
+					{
+						sha384.Add(key, item);
+					}
+					// If the file has a SHA-256
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA256))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA256)))
+					{
+						sha256.Add(key, item);
+					}
+					// If the file has a SHA-1
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1)))
+					{
+						sha1.Add(key, item);
+					}
+					// If the file has no SHA-1 but has an MD5
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).MD5))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).MD5)))
+					{
+						md5.Add(key, item);
+					}
+					// If the file has no MD5 but a CRC
+					else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1))
+						|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1)))
+					{
+						crc.Add(key, item);
+					}
+					else
+					{
+						other.Add(key, item);
+					}
+				}
+			});
+
+			// Get the output directory
+			if (outDir != "")
+			{
+				outDir = outDir + Path.GetDirectoryName(this.FileName).Remove(0, basepath.Length - 1);
+			}
+			else
+			{
+				outDir = Path.GetDirectoryName(this.FileName);
+			}
+
+			// Now, output all of the files to the output directory
+			Globals.Logger.User("DAT information created, outputting new files");
+			bool success = true;
+			success &= nodump.WriteToFile(outDir);
+			success &= sha512.WriteToFile(outDir);
+			success &= sha384.WriteToFile(outDir);
+			success &= sha256.WriteToFile(outDir);
+			success &= sha1.WriteToFile(outDir);
+			success &= md5.WriteToFile(outDir);
+			success &= crc.WriteToFile(outDir);
+
+			return success;
+		}
+
+		/// <summary>
+		/// Split a SuperDAT by lowest available directory level
+		/// </summary>
+		/// <param name="outDir">Name of the directory to write the DATs out to</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <param name="shortname">True if short names should be used, false otherwise</param>
+		/// <param name="basedat">True if original filenames should be used as the base for output filename, false otherwise</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		public bool SplitByLevel(string outDir, string basepath, bool shortname, bool basedat)
+		{
+			// Sanitize the basepath to be more predictable
+			basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
+
+			// First, organize by games so that we can do the right thing
+			BucketBy(SortedBy.Game, DedupeType.None, lower: false, norename: true);
+
+			// Create a temporary DAT to add things to
+			DatFile tempDat = new DatFile(this)
+			{
+				Name = null,
+			};
+
+			// Sort the input keys
+			List<string> keys = Keys.ToList();
+			keys.Sort(SplitByLevelSort);
+
+			// Then, we loop over the games
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				// Here, the key is the name of the game to be used for comparison
+				if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key))
+				{
+					// Process and output the DAT
+					SplitByLevelHelper(tempDat, outDir, shortname, basedat);
+
+					// Reset the DAT for the next items
+					tempDat = new DatFile(this)
+					{
+						Name = null,
+					};
+				}
+
+				// Clean the input list and set all games to be pathless
+				List<DatItem> items = this[key];
+				items.ForEach(item => item.MachineName = Style.GetFileName(item.MachineName));
+				items.ForEach(item => item.Description = Style.GetFileName(item.Description));
+
+				// Now add the game to the output DAT
+				tempDat.AddRange(key, items);
+
+				// Then set the DAT name to be the parent directory name
+				tempDat.Name = Style.GetDirectoryName(key);
+			});
+
+			// Then we write the last DAT out since it would be skipped otherwise
+			SplitByLevelHelper(tempDat, outDir, shortname, basedat);
+
+			return true;
+		}
+
+		/// <summary>
+		/// Helper function for SplitByLevel to sort the input game names
+		/// </summary>
+		/// <param name="a">First string to compare</param>
+		/// <param name="b">Second string to compare</param>
+		/// <returns>-1 for a coming before b, 0 for a == b, 1 for a coming after b</returns>
+		private int SplitByLevelSort(string a, string b)
+		{
+			NaturalComparer nc = new NaturalComparer();
+			int adeep = a.Count(c => c == '/' || c == '\\');
+			int bdeep = b.Count(c => c == '/' || c == '\\');
+
+			if (adeep == bdeep)
+			{
+				return nc.Compare(a, b);
+			}
+			return adeep - bdeep;
+		}
+
+		/// <summary>
+		/// Helper function for SplitByLevel to clean and write out a DAT
+		/// </summary>
+		/// <param name="datFile">DAT to clean and write out</param>
+		/// <param name="outDir">Directory to write out to</param>
+		/// <param name="shortname">True if short naming scheme should be used, false otherwise</param>
+		/// <param name="restore">True if original filenames should be used as the base for output filename, false otherwise</param>
+		private void SplitByLevelHelper(DatFile datFile, string outDir, bool shortname, bool restore)
+		{
+			// Get the name from the DAT to use separately
+			string name = datFile.Name;
+			string expName = name.Replace("/", " - ").Replace("\\", " - ");
+
+			// Get the path that the file will be written out to
+			string path = HttpUtility.HtmlDecode(String.IsNullOrEmpty(name)
+				? outDir
+				: Path.Combine(outDir, name));
+
+			// Now set the new output values
+			datFile.FileName = HttpUtility.HtmlDecode(String.IsNullOrEmpty(name)
+				? FileName
+				: (shortname
+					? Style.GetFileName(name)
+					: expName
+					)
+				);
+			datFile.FileName = (restore ? FileName + " (" + datFile.FileName + ")" : datFile.FileName);
+			datFile.Name = Name + " (" + expName + ")";
+			datFile.Description = (String.IsNullOrEmpty(Description) ? datFile.Name : Description + " (" + expName + ")");
+			datFile.Type = null;
+
+			// Write out the temporary DAT to the proper directory
+			datFile.WriteToFile(path);
+		}
+
+		/// <summary>
+		/// Split a DAT by type of Rom
+		/// </summary>
+		/// <param name="outDir">Name of the directory to write the DATs out to</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		public bool SplitByType(string outDir, string basepath)
+		{
+			// Sanitize the basepath to be more predictable
+			basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
+
+			// Create each of the respective output DATs
+			Globals.Logger.User("Creating and populating new DATs");
+			DatFile romdat = new DatFile
+			{
+				FileName = this.FileName + " (ROM)",
+				Name = this.Name + " (ROM)",
+				Description = this.Description + " (ROM)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile diskdat = new DatFile
+			{
+				FileName = this.FileName + " (Disk)",
+				Name = this.Name + " (Disk)",
+				Description = this.Description + " (Disk)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+			DatFile sampledat = new DatFile
+			{
+				FileName = this.FileName + " (Sample)",
+				Name = this.Name + " (Sample)",
+				Description = this.Description + " (Sample)",
+				Category = this.Category,
+				Version = this.Version,
+				Date = this.Date,
+				Author = this.Author,
+				Email = this.Email,
+				Homepage = this.Homepage,
+				Url = this.Url,
+				Comment = this.Comment,
+				Header = this.Header,
+				Type = this.Type,
+				ForceMerging = this.ForceMerging,
+				ForceNodump = this.ForceNodump,
+				ForcePacking = this.ForcePacking,
+				DatFormat = this.DatFormat,
+				DedupeRoms = this.DedupeRoms,
+			};
+
+			// Now populate each of the DAT objects in turn
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = this[key];
+				foreach(DatItem item in items)
+				{
+					// If the file is a Rom
+					if (item.Type == ItemType.Rom)
+					{
+						romdat.Add(key, item);
+					}
+					// If the file is a Disk
+					else if (item.Type == ItemType.Disk)
+					{
+						diskdat.Add(key, item);
+					}
+					// If the file is a Sample
+					else if (item.Type == ItemType.Sample)
+					{
+						sampledat.Add(key, item);
+					}
+				}
+			});
+
+			// Get the output directory
+			if (outDir != "")
+			{
+				outDir = outDir + Path.GetDirectoryName(this.FileName).Remove(0, basepath.Length - 1);
+			}
+			else
+			{
+				outDir = Path.GetDirectoryName(this.FileName);
+			}
+
+			// Now, output all of the files to the output directory
+			Globals.Logger.User("DAT information created, outputting new files");
+			bool success = true;
+			success &= romdat.WriteToFile(outDir);
+			success &= diskdat.WriteToFile(outDir);
+			success &= sampledat.WriteToFile(outDir);
+
+			return success;
+		}
+
+		#endregion
+	}
+}

SabreTools.Library/DatFiles/DatFile.Statistics.cs

@@ -0,0 +1,639 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Web;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+using SabreTools.Library.Tools;
+
+#if MONO
+using System.IO;
+#else
+using Alphaleonis.Win32.Filesystem;
+
+using SearchOption = System.IO.SearchOption;
+using StreamWriter = System.IO.StreamWriter;
+#endif
+
+namespace SabreTools.Library.DatFiles
+{
+	/*
+	 * TODO: Make output standard width (HTML, without making the entire thing a table)
+	 * TODO: Multithreading? Either StringBuilder or locking
+	 */
+	public partial class DatFile
+	{
+		#region Instance Methods
+
+		#region Statistics
+
+		/// <summary>
+		/// Recalculate the statistics for the Dat
+		/// </summary>
+		public void RecalculateStats()
+		{
+			// Wipe out any stats already there
+			_datStats.Reset();
+
+			// If we have a blank Dat in any way, return
+			if (this == null || Count == 0)
+			{
+				return;
+			}
+
+			// Loop through and add
+			List<string> keys = Keys.ToList();
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
+			{
+				List<DatItem> items = this[key];
+				foreach(DatItem item in items)
+				{
+					_datStats.AddItem(item);
+				}
+			});
+		}
+
+		/// <summary>
+		/// Output the stats for the Dat in a human-readable format
+		/// </summary>
+		/// <param name="outputs">Dictionary representing the outputs</param>
+		/// <param name="statDatFormat">Set the statistics output format to use</param>
+		/// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param>
+		/// <param name="game">Number of games to use, -1 means recalculate games (default)</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise (default)</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise (default)</param>
+		public void OutputStats(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat,
+			bool recalculate = false, long game = -1, bool baddumpCol = false, bool nodumpCol = false)
+		{
+			// If we're supposed to recalculate the statistics, do so
+			if (recalculate)
+			{
+				RecalculateStats();
+			}
+
+			BucketBy(SortedBy.Game, DedupeType.None, norename: true);
+			if (TotalSize < 0)
+			{
+				TotalSize = Int64.MaxValue + TotalSize;
+			}
+
+			// Log the results to screen
+			string results = @"For '" + FileName + @"':
+--------------------------------------------------
+    Uncompressed size:       " + Style.GetBytesReadable(TotalSize) + @"
+    Games found:             " + (game == -1 ? Keys.Count() : game) + @"
+    Roms found:              " + RomCount + @"
+    Disks found:             " + DiskCount + @"
+    Roms with CRC:           " + CRCCount + @"
+    Roms with MD5:           " + MD5Count + @"
+    Roms with SHA-1:         " + SHA1Count + @"
+    Roms with SHA-256:       " + SHA256Count + @"
+    Roms with SHA-384:       " + SHA384Count + @"
+    Roms with SHA-512:       " + SHA512Count + "\n";
+
+			if (baddumpCol)
+			{
+				results += "	Roms with BadDump status: " + BaddumpCount + "\n";
+			}
+			if (nodumpCol)
+			{
+				results += "	Roms with Nodump status: " + NodumpCount + "\n";
+			}
+
+			// For spacing between DATs
+			results += "\n\n";
+
+			Globals.Logger.User(results);
+
+			// Now write it out to file as well
+			string line = "";
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				line = @"'" + FileName + @"':
+--------------------------------------------------
+    Uncompressed size:       " + Style.GetBytesReadable(TotalSize) + @"
+    Games found:             " + (game == -1 ? Keys.Count() : game) + @"
+    Roms found:              " + RomCount + @"
+    Disks found:             " + DiskCount + @"
+    Roms with CRC:           " + CRCCount + @"
+    Roms with SHA-1:         " + SHA1Count + @"
+    Roms with SHA-256:       " + SHA256Count + @"
+    Roms with SHA-384:       " + SHA384Count + @"
+    Roms with SHA-512:       " + SHA512Count + "\n";
+
+				if (baddumpCol)
+				{
+					line += "	Roms with BadDump status: " + BaddumpCount + "\n";
+				}
+				if (nodumpCol)
+				{
+					line += "	Roms with Nodump status: " + NodumpCount + "\n";
+				}
+
+				// For spacing between DATs
+				line += "\n\n";
+
+				outputs[StatDatFormat.None].Write(line);
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				line = "\"" + FileName + "\","
+					+ "\"" + TotalSize + "\","
+					+ "\"" + (game == -1 ? Keys.Count() : game) + "\","
+					+ "\"" + RomCount + "\","
+					+ "\"" + DiskCount + "\","
+					+ "\"" + CRCCount + "\","
+					+ "\"" + MD5Count + "\","
+					+ "\"" + SHA1Count + "\","
+					+ "\"" + SHA256Count + "\","
+					+ "\"" + SHA384Count + "\","
+					+ "\"" + SHA512Count + "\"";
+
+				if (baddumpCol)
+				{
+					line += ",\"" + BaddumpCount + "\"";
+				}
+				if (nodumpCol)
+				{
+					line += ",\"" + NodumpCount + "\"";
+				}
+
+				line += "\n";
+				outputs[StatDatFormat.CSV].Write(line);
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				line = "\t\t\t<tr" + (FileName.StartsWith("DIR: ")
+							? " class=\"dir\"><td>" + HttpUtility.HtmlEncode(FileName.Remove(0, 5))
+							: "><td>" + HttpUtility.HtmlEncode(FileName)) + "</td>"
+						+ "<td align=\"right\">" + Style.GetBytesReadable(TotalSize) + "</td>"
+						+ "<td align=\"right\">" + (game == -1 ? Keys.Count() : game) + "</td>"
+						+ "<td align=\"right\">" + RomCount + "</td>"
+						+ "<td align=\"right\">" + DiskCount + "</td>"
+						+ "<td align=\"right\">" + CRCCount + "</td>"
+						+ "<td align=\"right\">" + MD5Count + "</td>"
+						+ "<td align=\"right\">" + SHA1Count + "</td>"
+						+ "<td align=\"right\">" + SHA256Count + "</td>";
+
+				if (baddumpCol)
+				{
+					line += "<td align=\"right\">" + BaddumpCount + "</td>";
+				}
+				if (nodumpCol)
+				{
+					line += "<td align=\"right\">" + NodumpCount + "</td>";
+				}
+
+				line += "</tr>\n";
+				outputs[StatDatFormat.HTML].Write(line);
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				line = "\"" + FileName + "\"\t"
+						+ "\"" + TotalSize + "\"\t"
+						+ "\"" + (game == -1 ? Keys.Count() : game) + "\"\t"
+						+ "\"" + RomCount + "\"\t"
+						+ "\"" + DiskCount + "\"\t"
+						+ "\"" + CRCCount + "\"\t"
+						+ "\"" + MD5Count + "\"\t"
+						+ "\"" + SHA1Count + "\"\t"
+						+ "\"" + SHA256Count + "\"\t"
+						+ "\"" + SHA384Count + "\"\t"
+						+ "\"" + SHA512Count + "\"";
+
+				if (baddumpCol)
+				{
+					line += "\t\"" + BaddumpCount + "\"";
+				}
+				if (nodumpCol)
+				{
+					line += "\t\"" + NodumpCount + "\"";
+				}
+
+				line += "\n";
+				outputs[StatDatFormat.TSV].Write(line);
+			}
+		}
+
+		#endregion
+
+		#endregion // Instance Methods
+
+		#region Static Methods
+
+		#region Statistics
+
+		/// <summary>
+		/// Output the stats for a list of input dats as files in a human-readable format
+		/// </summary>
+		/// <param name="inputs">List of input files and folders</param>
+		/// <param name="reportName">Name of the output file</param>
+		/// <param name="single">True if single DAT stats are output, false otherwise</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
+		/// <param name="statDatFormat" > Set the statistics output format to use</param>
+		public static void OutputStats(List<string> inputs, string reportName, string outDir, bool single,
+			bool baddumpCol, bool nodumpCol, StatDatFormat statDatFormat)
+		{
+			// If there's no output format, set the default
+			if (statDatFormat == 0x0)
+			{
+				statDatFormat = StatDatFormat.None;
+			}
+
+			// Get the proper output file name
+			if (String.IsNullOrEmpty(reportName))
+			{
+				reportName = "report";
+			}
+			outDir = Path.GetFullPath(outDir);
+
+			// Get the dictionary of desired outputs
+			Dictionary<StatDatFormat, StreamWriter> outputs = OutputStatsGetOutputWriters(statDatFormat, reportName, outDir);
+
+			// Make sure we have all files
+			List<Tuple<string, string>> newinputs = new List<Tuple<string, string>>(); // item, basepath
+			Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
+			{
+				if (File.Exists(input))
+				{
+					lock (newinputs)
+					{
+						newinputs.Add(Tuple.Create(Path.GetFullPath(input), Path.GetDirectoryName(Path.GetFullPath(input))));
+					}
+				}
+				if (Directory.Exists(input))
+				{
+					foreach (string file in Directory.GetFiles(input, "*", SearchOption.AllDirectories))
+					{
+						lock (newinputs)
+						{
+							newinputs.Add(Tuple.Create(Path.GetFullPath(file), Path.GetFullPath(input)));
+						}
+					}
+				}
+			});
+
+			newinputs = newinputs
+				.OrderBy(i => Path.GetDirectoryName(i.Item1))
+				.ThenBy(i => Path.GetFileName(i.Item1))
+				.ToList();
+
+			// Write the header, if any
+			OutputStatsWriteHeader(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+			// Init all total variables
+			DatStats totalStats = new DatStats();
+			long totalGame = 0;
+
+			// Init directory-level variables
+			string lastdir = null;
+			string basepath = null;
+			DatStats dirStats = new DatStats();
+			long dirGame = 0;
+			
+			// Now process each of the input files
+			foreach (Tuple<string, string> filename in newinputs)
+			{
+				// Get the directory for the current file
+				string thisdir = Path.GetDirectoryName(filename.Item1);
+				basepath = Path.GetDirectoryName(filename.Item2);
+
+				// If we don't have the first file and the directory has changed, show the previous directory stats and reset
+				if (lastdir != null && thisdir != lastdir)
+				{
+					// Output separator if needed
+					OutputStatsWriteMidSeparator(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+					DatFile lastdirdat = new DatFile
+					{
+						FileName = "DIR: " + HttpUtility.HtmlEncode(lastdir.Remove(0, basepath.Length + (basepath.Length == 0 ? 0 : 1))),
+						_datStats = dirStats,
+					};
+
+					lastdirdat.OutputStats(outputs, statDatFormat,
+						game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
+
+					// Write the mid-footer, if any
+					OutputStatsWriteMidFooter(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+					// Write the header, if any
+					OutputStatsWriteMidHeader(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+					// Reset the directory stats
+					dirStats.Reset();
+					dirGame = 0;
+				}
+
+				Globals.Logger.Verbose("Beginning stat collection for '{0}'", false, filename.Item1);
+				List<string> games = new List<string>();
+				DatFile datdata = new DatFile();
+				datdata.Parse(filename.Item1, 0, 0);
+				datdata.BucketBy(SortedBy.Game, DedupeType.None, norename: true);
+
+				// Output single DAT stats (if asked)
+				Globals.Logger.User("Adding stats for file '{0}'\n", false, filename.Item1);
+				if (single)
+				{
+					datdata.OutputStats(outputs, statDatFormat,
+						baddumpCol: baddumpCol, nodumpCol: nodumpCol);
+				}
+
+				// Add single DAT stats to dir
+				dirStats.AddStats(datdata._datStats);
+				dirGame += datdata.Keys.Count();
+
+				// Add single DAT stats to totals
+				totalStats.AddStats(datdata._datStats);
+				totalGame += datdata.Keys.Count();
+
+				// Make sure to assign the new directory
+				lastdir = thisdir;
+			}
+
+			// Output the directory stats one last time
+			OutputStatsWriteMidSeparator(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+			if (single)
+			{
+				DatFile dirdat = new DatFile
+				{
+					FileName = "DIR: " + HttpUtility.HtmlEncode(lastdir.Remove(0, basepath.Length + (basepath.Length == 0 ? 0 : 1))),
+					_datStats = dirStats,
+				};
+
+				dirdat.OutputStats(outputs, statDatFormat,
+					game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
+			}
+
+			// Write the mid-footer, if any
+			OutputStatsWriteMidFooter(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+			// Write the header, if any
+			OutputStatsWriteMidHeader(outputs, statDatFormat, baddumpCol, nodumpCol);
+
+			// Reset the directory stats
+			dirStats.Reset();
+			dirGame = 0;
+
+			// Output total DAT stats
+			DatFile totaldata = new DatFile
+			{
+				FileName = "DIR: All DATs",
+				_datStats = totalStats,
+			};
+
+			totaldata.OutputStats(outputs, statDatFormat,
+				game: totalGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
+
+			// Output footer if needed
+			OutputStatsWriteFooter(outputs, statDatFormat);
+
+			// Flush and dispose of the stream writers
+			foreach (StatDatFormat format in outputs.Keys)
+			{
+				outputs[format].Flush();
+				outputs[format].Dispose();
+			}
+
+			Globals.Logger.User(@"
+Please check the log folder if the stats scrolled offscreen", false);
+		}
+
+		/// <summary>
+		/// Get the proper extension for the stat output format
+		/// </summary>
+		/// <param name="statDatFormat">StatDatFormat to get the extension for</param>
+		/// <param name="reportName">Name of the input file to use</param>
+		/// <param name="outDir">Output path to use</param>
+		/// <returns>Dictionary of file types to StreamWriters</returns>
+		private static Dictionary<StatDatFormat, StreamWriter> OutputStatsGetOutputWriters(StatDatFormat statDatFormat, string reportName, string outDir)
+		{
+			Dictionary<StatDatFormat, StreamWriter> output = new Dictionary<StatDatFormat, StreamWriter>();
+
+			// First try to create the output directory if we need to
+			if (!Directory.Exists(outDir))
+			{
+				Directory.CreateDirectory(outDir);
+			}
+
+			// For each output format, get the appropriate stream writer
+			if ((statDatFormat & StatDatFormat.None) != 0)
+			{
+				reportName = Style.GetFileNameWithoutExtension(reportName) + ".txt";
+				reportName = Path.Combine(outDir, reportName);
+
+				// Create the StreamWriter for this file
+				output.Add(StatDatFormat.None, new StreamWriter(FileTools.TryCreate(reportName)));
+			}
+			if ((statDatFormat & StatDatFormat.CSV) != 0)
+			{
+				reportName = Style.GetFileNameWithoutExtension(reportName) + ".csv";
+				reportName = Path.Combine(outDir, reportName);
+
+				// Create the StreamWriter for this file
+				output.Add(StatDatFormat.CSV, new StreamWriter(FileTools.TryCreate(reportName)));
+			}
+			if ((statDatFormat & StatDatFormat.HTML) != 0)
+			{
+				reportName = Style.GetFileNameWithoutExtension(reportName) + ".html";
+				reportName = Path.Combine(outDir, reportName);
+
+				// Create the StreamWriter for this file
+				output.Add(StatDatFormat.HTML, new StreamWriter(FileTools.TryCreate(reportName)));
+			}
+			if ((statDatFormat & StatDatFormat.TSV) != 0)
+			{
+				reportName = Style.GetFileNameWithoutExtension(reportName) + ".csv";
+				reportName = Path.Combine(outDir, reportName);
+
+				// Create the StreamWriter for this file
+				output.Add(StatDatFormat.TSV, new StreamWriter(FileTools.TryCreate(reportName)));
+			}
+
+			return output;
+		}
+
+		/// <summary>
+		/// Write out the header to the stream, if any exists
+		/// </summary>
+		/// <param name="outputs">Dictionary representing the outputs</param>
+		/// <param name="statDatFormat">StatDatFormat representing output format</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
+		private static void OutputStatsWriteHeader(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, bool baddumpCol, bool nodumpCol)
+		{
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				outputs[StatDatFormat.CSV].Write("\"File Name\",\"Total Size\",\"Games\",\"Roms\",\"Disks\",\"# with CRC\",\"# with MD5\",\"# with SHA-1\",\"# with SHA-256\""
+					+ (baddumpCol ? ",\"BadDumps\"" : "") + (nodumpCol ? ",\"Nodumps\"" : "") + "\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				outputs[StatDatFormat.HTML].Write(@"<!DOCTYPE html>
+<html>
+	<header>
+		<title>DAT Statistics Report</title>
+		<style>
+			body {
+				background-color: lightgray;
+			}
+			.dir {
+				color: #0088FF;
+			}
+			.right {
+				align: right;
+			}
+		</style>
+	</header>
+	<body>
+		<h2>DAT Statistics Report (" + DateTime.Now.ToShortDateString() + @")</h2>
+		<table border=""1"" cellpadding=""5"" cellspacing=""0"">
+");
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				outputs[StatDatFormat.TSV].Write("\"File Name\"\t\"Total Size\"\t\"Games\"\t\"Roms\"\t\"Disks\"\t\"# with CRC\"\t\"# with MD5\"\t\"# with SHA-1\"\t\"# with SHA-256\""
+						+ (baddumpCol ? "\t\"BadDumps\"" : "") + (nodumpCol ? "\t\"Nodumps\"" : "") + "\n");
+			}
+
+			// Now write the mid header for those who need it
+			OutputStatsWriteMidHeader(outputs, statDatFormat, baddumpCol, nodumpCol);
+		}
+
+		/// <summary>
+		/// Write out the mid-header to the stream, if any exists
+		/// </summary>
+		/// <param name="outputs">Dictionary representing the outputs</param>
+		/// <param name="statDatFormat">StatDatFormat representing output format</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
+		private static void OutputStatsWriteMidHeader(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, bool baddumpCol, bool nodumpCol)
+		{
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				outputs[StatDatFormat.HTML].Write(@"			<tr bgcolor=""gray""><th>File Name</th><th align=""right"">Total Size</th><th align=""right"">Games</th><th align=""right"">Roms</th>"
++ @"<th align=""right"">Disks</th><th align=""right"">&#35; with CRC</th><th align=""right"">&#35; with MD5</th><th align=""right"">&#35; with SHA-1</th><th align=""right"">&#35; with SHA-256</th>"
++ (baddumpCol ? "<th class=\".right\">Baddumps</th>" : "") + (nodumpCol ? "<th class=\".right\">Nodumps</th>" : "") + "</tr>\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				// Nothing
+			}
+		}
+
+		/// <summary>
+		/// Write out the separator to the stream, if any exists
+		/// </summary>
+		/// <param name="outputs">Dictionary representing the outputs</param>
+		/// <param name="statDatFormat">StatDatFormat representing output format</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
+		private static void OutputStatsWriteMidSeparator(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, bool baddumpCol, bool nodumpCol)
+		{
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				outputs[StatDatFormat.HTML].Write("<tr><td colspan=\""
+						+ (baddumpCol && nodumpCol
+							? "12"
+							: (baddumpCol ^ nodumpCol
+								? "11"
+								: "10")
+							)
+						+ "\"></td></tr>\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				// Nothing
+			}
+		}
+
+		/// <summary>
+		/// Write out the footer-separator to the stream, if any exists
+		/// </summary>
+		/// <param name="outputs">Dictionary representing the outputs</param>
+		/// <param name="statDatFormat">StatDatFormat representing output format</param>
+		/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
+		/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
+		private static void OutputStatsWriteMidFooter(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, bool baddumpCol, bool nodumpCol)
+		{
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				outputs[StatDatFormat.None].Write("\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				outputs[StatDatFormat.CSV].Write("\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				outputs[StatDatFormat.HTML].Write("<tr border=\"0\"><td colspan=\""
+						+ (baddumpCol && nodumpCol
+							? "12"
+							: (baddumpCol ^ nodumpCol
+								? "11"
+								: "10")
+							)
+						+ "\"></td></tr>\n");
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				outputs[StatDatFormat.TSV].Write("\n");
+			}
+		}
+
+		/// <summary>
+		/// Write out the footer to the stream, if any exists
+		/// </summary>
+		/// <param name="sw">StreamWriter representing the output</param>
+		/// <param name="statDatFormat">StatDatFormat representing output format</param>
+		private static void OutputStatsWriteFooter(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat)
+		{
+			if (outputs.ContainsKey(StatDatFormat.None))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.CSV))
+			{
+				// Nothing
+			}
+			if (outputs.ContainsKey(StatDatFormat.HTML))
+			{
+				outputs[StatDatFormat.HTML].Write(@"		</table>
+	</body>
+</html>
+");
+			}
+			if (outputs.ContainsKey(StatDatFormat.TSV))
+			{
+				// Nothing
+			}
+		}
+
+		#endregion
+
+		#endregion // Static Methods
+	}
+}

SabreTools.Library/DatFiles/DatHeader.cs

@@ -0,0 +1,278 @@
+using System;
+using System.Collections.Generic;
+
+using SabreTools.Library.Data;
+
+namespace SabreTools.Library.DatFiles
+{
+	public class DatHeader : ICloneable
+	{
+		#region Private instance variables
+
+		// Data common to most DAT types
+		private string _fileName;
+		private string _name;
+		private string _description;
+		private string _rootDir;
+		private string _category;
+		private string _version;
+		private string _date;
+		private string _author;
+		private string _email;
+		private string _homepage;
+		private string _url;
+		private string _comment;
+		private string _header;
+		private string _type; // Generally only used for SuperDAT
+		private ForceMerging _forceMerging;
+		private ForceNodump _forceNodump;
+		private ForcePacking _forcePacking;
+		private DatFormat _datFormat;
+		private bool _excludeOf;
+		private DedupeType _dedupeRoms;
+		private Hash _stripHash;
+		private bool _oneGameOneRegion;
+		private List<string> _regions;
+
+		// Data specific to the Miss DAT type
+		private bool _useGame;
+		private string _prefix;
+		private string _postfix;
+		private bool _quotes;
+		private string _repExt;
+		private string _addExt;
+		private bool _remExt;
+		private bool _gameName;
+		private bool _romba;
+
+		#endregion
+
+		#region Publicly facing variables
+
+		// Data common to most DAT types
+		public string FileName
+		{
+			get { return _fileName; }
+			set { _fileName = value; }
+		}
+		public string Name
+		{
+			get { return _name; }
+			set { _name = value; }
+		}
+		public string Description
+		{
+			get { return _description; }
+			set { _description = value; }
+		}
+		public string RootDir
+		{
+			get { return _rootDir; }
+			set { _rootDir = value; }
+		}
+		public string Category
+		{
+			get { return _category; }
+			set { _category = value; }
+		}
+		public string Version
+		{
+			get { return _version; }
+			set { _version = value; }
+		}
+		public string Date
+		{
+			get { return _date; }
+			set { _date = value; }
+		}
+		public string Author
+		{
+			get { return _author; }
+			set { _author = value; }
+		}
+		public string Email
+		{
+			get { return _email; }
+			set { _email = value; }
+		}
+		public string Homepage
+		{
+			get { return _homepage; }
+			set { _homepage = value; }
+		}
+		public string Url
+		{
+			get { return _url; }
+			set { _url = value; }
+		}
+		public string Comment
+		{
+			get { return _comment; }
+			set { _comment = value; }
+		}
+		public string Header
+		{
+			get { return _header; }
+			set { _header = value; }
+		}
+		public string Type // Generally only used for SuperDAT
+		{
+			get { return _type; }
+			set { _type = value; }
+		}
+		public ForceMerging ForceMerging
+		{
+			get { return _forceMerging; }
+			set { _forceMerging = value; }
+		}
+		public ForceNodump ForceNodump
+		{
+			get { return _forceNodump; }
+			set { _forceNodump = value; }
+		}
+		public ForcePacking ForcePacking
+		{
+			get { return _forcePacking; }
+			set { _forcePacking = value; }
+		}
+		public DatFormat DatFormat
+		{
+			get { return _datFormat; }
+			set { _datFormat = value; }
+		}
+		public bool ExcludeOf
+		{
+			get { return _excludeOf; }
+			set { _excludeOf = value; }
+		}
+		public DedupeType DedupeRoms
+		{
+			get { return _dedupeRoms; }
+			set { _dedupeRoms = value; }
+		}
+		public Hash StripHash
+		{
+			get { return _stripHash; }
+			set { _stripHash = value; }
+		}
+		public bool OneGameOneRegion
+		{
+			get { return _oneGameOneRegion; }
+			set { _oneGameOneRegion = value; }
+		}
+		public List<string> Regions
+		{
+			get
+			{
+				if (_regions == null)
+				{
+					_regions = new List<string>();
+				}
+
+				return _regions;
+			}
+			set { _regions = value; }
+		}
+
+		// Data specific to the Miss DAT type
+		public bool UseGame
+		{
+			get { return _useGame; }
+			set { _useGame = value; }
+		}
+		public string Prefix
+		{
+			get { return _prefix; }
+			set { _prefix = value; }
+		}
+		public string Postfix
+		{
+			get { return _postfix; }
+			set { _postfix = value; }
+		}
+		public bool Quotes
+		{
+			get { return _quotes; }
+			set { _quotes = value; }
+		}
+		public string RepExt
+		{
+			get { return _repExt; }
+			set { _repExt = value; }
+		}
+		public string AddExt
+		{
+			get { return _addExt; }
+			set { _addExt = value; }
+		}
+		public bool RemExt
+		{
+			get { return _remExt; }
+			set { _remExt = value; }
+		}
+		public bool GameName
+		{
+			get { return _gameName; }
+			set { _gameName = value; }
+		}
+		public bool Romba
+		{
+			get { return _romba; }
+			set { _romba = value; }
+		}
+
+		#endregion
+
+		#region Instance Methods
+
+		#region Cloning Methods
+
+		/// <summary>
+		/// Clone the current header
+		/// </summary>
+		/// <returns></returns>
+		public object Clone()
+		{
+			return new DatHeader()
+			{
+				_fileName = this._fileName,
+				_name = this._name,
+				_description = this._description,
+				_rootDir = this._rootDir,
+				_category = this._category,
+				_version = this._version,
+				_date = this._date,
+				_author = this._author,
+				_email = this._email,
+				_homepage = this._homepage,
+				_url = this._url,
+				_comment = this._comment,
+				_header = this._header,
+				_type = this._type,
+				_forceMerging = this._forceMerging,
+				_forceNodump = this._forceNodump,
+				_forcePacking = this._forcePacking,
+				_datFormat = this._datFormat,
+				_excludeOf = this._excludeOf,
+				_dedupeRoms = this._dedupeRoms,
+				_stripHash = this._stripHash,
+				_oneGameOneRegion = this._oneGameOneRegion,
+				_regions = this._regions,
+
+				_useGame = this._useGame,
+				_prefix = this._prefix,
+				_postfix = this._postfix,
+				_quotes = this._quotes,
+				_repExt = this._repExt,
+				_addExt = this._addExt,
+				_remExt = this._remExt,
+				_gameName = this._gameName,
+				_romba = this._romba,
+			};
+		}
+
+		#endregion
+
+		#endregion // Instance Methods
+	}
+}

SabreTools.Library/DatFiles/DatStats.cs

@@ -0,0 +1,344 @@
+using System;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+
+namespace SabreTools.Library.DatFiles
+{
+	public class DatStats
+	{
+		#region Private instance variables
+
+		// Object used to lock stats updates
+		private object _lockObject = new object();
+
+		// Overall item count
+		private long _count = 0;
+
+		// Individual DatItem type counts
+		private long _archiveCount = 0;
+		private long _biosSetCount = 0;
+		private long _diskCount = 0;
+		private long _releaseCount = 0;
+		private long _romCount = 0;
+		private long _sampleCount = 0;
+
+		// Total reported size
+		private long _totalSize = 0;
+
+		// Individual hash counts
+		private long _crcCount = 0;
+		private long _md5Count = 0;
+		private long _sha1Count = 0;
+		private long _sha256Count = 0;
+		private long _sha384Count = 0;
+		private long _sha512Count = 0;
+
+		// Individual status counts
+		private long _baddumpCount = 0;
+		private long _goodCount = 0;
+		private long _nodumpCount = 0;
+		private long _verifiedCount = 0;
+
+		#endregion
+
+		#region Publicly facing variables
+
+		// Overall item count
+		public long Count
+		{
+			get { return _count; }
+			set { _count = value; }
+		}
+
+		// Individual DatItem type counts
+		public long ArchiveCount
+		{
+			get { return _archiveCount; }
+			set { _archiveCount = value; }
+		}
+		public long BiosSetCount
+		{
+			get { return _biosSetCount; }
+			set { _biosSetCount = value; }
+		}
+		public long DiskCount
+		{
+			get { return _diskCount; }
+			set { _diskCount = value; }
+		}
+		public long ReleaseCount
+		{
+			get { return _releaseCount; }
+			set { _releaseCount = value; }
+		}
+		public long RomCount
+		{
+			get { return _romCount; }
+			set { _romCount = value; }
+		}
+		public long SampleCount
+		{
+			get { return _sampleCount; }
+			set { _sampleCount = value; }
+		}
+
+		// Total reported size
+		public long TotalSize
+		{
+			get { return _totalSize; }
+			set { _totalSize = value; }
+		}
+
+		// Individual hash counts
+		public long CRCCount
+		{
+			get { return _crcCount; }
+			set { _crcCount = value; }
+		}
+		public long MD5Count
+		{
+			get { return _md5Count; }
+			set { _md5Count = value; }
+		}
+		public long SHA1Count
+		{
+			get { return _sha1Count; }
+			set { _sha1Count = value; }
+		}
+		public long SHA256Count
+		{
+			get { return _sha256Count; }
+			set { _sha256Count = value; }
+		}
+		public long SHA384Count
+		{
+			get { return _sha384Count; }
+			set { _sha384Count = value; }
+		}
+		public long SHA512Count
+		{
+			get { return _sha512Count; }
+			set { _sha512Count = value; }
+		}
+
+		// Individual status counts
+		public long BaddumpCount
+		{
+			get { return _baddumpCount; }
+			set { _baddumpCount = value; }
+		}
+		public long GoodCount
+		{
+			get { return _goodCount; }
+			set { _goodCount = value; }
+		}
+		public long NodumpCount
+		{
+			get { return _nodumpCount; }
+			set { _nodumpCount = value; }
+		}
+		public long VerifiedCount
+		{
+			get { return _verifiedCount; }
+			set { _verifiedCount = value; }
+		}
+
+		#endregion
+
+		#region Instance Methods
+
+		/// <summary>
+		/// Add to the statistics given a DatItem
+		/// </summary>
+		/// <param name="item">Item to add info from</param>
+		public void AddItem(DatItem item)
+		{
+			// No matter what the item is, we increate the count
+			lock (_lockObject)
+			{
+				_count += 1;
+
+				// Now we do different things for each item type
+
+				switch (item.Type)
+				{
+					case ItemType.Archive:
+						_archiveCount += 1;
+						break;
+					case ItemType.BiosSet:
+						_biosSetCount += 1;
+						break;
+					case ItemType.Disk:
+						_diskCount += 1;
+						if (((Disk)item).ItemStatus != ItemStatus.Nodump)
+						{
+							_md5Count += (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1);
+							_sha1Count += (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1);
+							_sha256Count += (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1);
+							_sha384Count += (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1);
+							_sha512Count += (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1);
+						}
+
+						_baddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
+						_goodCount += (((Disk)item).ItemStatus == ItemStatus.Good ? 1 : 0);
+						_nodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
+						_verifiedCount += (((Disk)item).ItemStatus == ItemStatus.Verified ? 1 : 0);
+						break;
+					case ItemType.Release:
+						_releaseCount += 1;
+						break;
+					case ItemType.Rom:
+						_romCount += 1;
+						if (((Rom)item).ItemStatus != ItemStatus.Nodump)
+						{
+							_totalSize += ((Rom)item).Size;
+							_crcCount += (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1);
+							_md5Count += (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1);
+							_sha1Count += (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1);
+							_sha256Count += (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1);
+							_sha384Count += (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1);
+							_sha512Count += (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1);
+						}
+
+						_baddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
+						_goodCount += (((Rom)item).ItemStatus == ItemStatus.Good ? 1 : 0);
+						_nodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
+						_verifiedCount += (((Rom)item).ItemStatus == ItemStatus.Verified ? 1 : 0);
+						break;
+					case ItemType.Sample:
+						_sampleCount += 1;
+						break;
+				}
+			}
+		}
+
+		/// <summary>
+		/// Add statistics from another DatStats object
+		/// </summary>
+		/// <param name="stats">DatStats object to add from</param>
+		public void AddStats(DatStats stats)
+		{
+			_count += stats.Count;
+
+			_archiveCount += stats.ArchiveCount;
+			_biosSetCount += stats.BiosSetCount;
+			_diskCount += stats.DiskCount;
+			_releaseCount += stats.ReleaseCount;
+			_romCount += stats.RomCount;
+			_sampleCount += stats.SampleCount;
+
+			_totalSize += stats.TotalSize;
+
+			// Individual hash counts
+			_crcCount += stats.CRCCount;
+			_md5Count += stats.MD5Count;
+			_sha1Count += stats.SHA1Count;
+			_sha256Count += stats.SHA256Count;
+			_sha384Count += stats.SHA384Count;
+			_sha512Count += stats.SHA512Count;
+
+			// Individual status counts
+			_baddumpCount += stats.BaddumpCount;
+			_goodCount += stats.GoodCount;
+			_nodumpCount += stats.NodumpCount;
+			_verifiedCount += stats.VerifiedCount;
+	}
+
+		/// <summary>
+		/// Remove from the statistics given a DatItem
+		/// </summary>
+		/// <param name="item">Item to remove info for</param>
+		public void RemoveItem(DatItem item)
+		{
+			// No matter what the item is, we increate the count
+			lock (_lockObject)
+			{
+				_count -= 1;
+
+				// Now we do different things for each item type
+
+				switch (item.Type)
+				{
+					case ItemType.Archive:
+						_archiveCount -= 1;
+						break;
+					case ItemType.BiosSet:
+						_biosSetCount -= 1;
+						break;
+					case ItemType.Disk:
+						_diskCount -= 1;
+						if (((Disk)item).ItemStatus != ItemStatus.Nodump)
+						{
+							_md5Count -= (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1);
+							_sha1Count -= (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1);
+							_sha256Count -= (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1);
+							_sha384Count -= (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1);
+							_sha512Count -= (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1);
+						}
+
+						_baddumpCount -= (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
+						_goodCount -= (((Disk)item).ItemStatus == ItemStatus.Good ? 1 : 0);
+						_nodumpCount -= (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
+						_verifiedCount -= (((Disk)item).ItemStatus == ItemStatus.Verified ? 1 : 0);
+						break;
+					case ItemType.Release:
+						_releaseCount -= 1;
+						break;
+					case ItemType.Rom:
+						_romCount -= 1;
+						if (((Rom)item).ItemStatus != ItemStatus.Nodump)
+						{
+							_totalSize -= ((Rom)item).Size;
+							_crcCount -= (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1);
+							_md5Count -= (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1);
+							_sha1Count -= (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1);
+							_sha256Count -= (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1);
+							_sha384Count -= (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1);
+							_sha512Count -= (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1);
+						}
+
+						_baddumpCount -= (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
+						_goodCount -= (((Rom)item).ItemStatus == ItemStatus.Good ? 1 : 0);
+						_nodumpCount -= (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
+						_verifiedCount -= (((Rom)item).ItemStatus == ItemStatus.Verified ? 1 : 0);
+						break;
+					case ItemType.Sample:
+						_sampleCount -= 1;
+						break;
+				}
+			}
+		}
+
+		/// <summary>
+		/// Reset all statistics
+		/// </summary>
+		public void Reset()
+		{
+			_count = 0;
+
+			_archiveCount = 0;
+			_biosSetCount = 0;
+			_diskCount = 0;
+			_releaseCount = 0;
+			_romCount = 0;
+			_sampleCount = 0;
+
+			_totalSize = 0;
+
+			_crcCount = 0;
+			_md5Count = 0;
+			_sha1Count = 0;
+			_sha256Count = 0;
+			_sha384Count = 0;
+			_sha512Count = 0;
+
+			_baddumpCount = 0;
+			_goodCount = 0;
+			_nodumpCount = 0;
+			_verifiedCount = 0;
+		}
+
+		#endregion // Instance Methods
+	}
+}

SabreTools.Library/DatFiles/Filter.cs

@@ -0,0 +1,710 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+using SabreTools.Library.Data;
+using SabreTools.Library.Items;
+
+namespace SabreTools.Library.DatFiles
+{
+	public class Filter
+	{
+		#region Private instance variables
+
+		#region Positive
+		
+		private List<string> _gameNames;
+		private List<string> _romNames;
+		private List<string> _romTypes;
+		private List<string> _crcs;
+		private List<string> _md5s;
+		private List<string> _sha1s;
+		private List<string> _sha256s;
+		private List<string> _sha384s;
+		private List<string> _sha512s;
+		private ItemStatus _itemStatuses;
+		private MachineType _machineTypes;
+
+		#endregion
+
+		#region Negative
+
+		private List<string> _notGameNames;
+		private List<string> _notRomNames;
+		private List<string> _notRomTypes;
+		private List<string> _notCrcs;
+		private List<string> _notMd5s;
+		private List<string> _notSha1s;
+		private List<string> _notSha256s;
+		private List<string> _notSha384s;
+		private List<string> _notSha512s;
+		private ItemStatus _itemNotStatuses;
+		private MachineType _machineNotTypes;
+
+		#endregion
+
+		#region Neutral
+
+		private long _sizeGreaterThanOrEqual;
+		private long _sizeLessThanOrEqual;
+		private long _sizeEqualTo;
+		private bool _includeOfInGame;
+		private bool? _runnable;
+
+		#endregion
+
+		#endregion // Private instance variables
+
+		#region Pubically facing variables
+
+		#region Positive
+
+		public List<string> GameNames
+		{
+			get { return _gameNames; }
+			set { _gameNames = value; }
+		}
+		public List<string> RomNames
+		{
+			get { return _romNames; }
+			set { _romNames = value; }
+		}
+		public List<string> RomTypes
+		{
+			get { return _romTypes; }
+			set { _romTypes = value; }
+		}
+		public List<string> CRCs
+		{
+			get { return _crcs; }
+			set { _crcs = value; }
+		}
+		public List<string> MD5s
+		{
+			get { return _md5s; }
+			set { _md5s = value; }
+		}
+		public List<string> SHA1s
+		{
+			get { return _sha1s; }
+			set { _sha1s = value; }
+		}
+		public List<string> SHA256s
+		{
+			get { return _sha256s; }
+			set { _sha256s = value; }
+		}
+		public List<string> SHA384s
+		{
+			get { return _sha384s; }
+			set { _sha384s = value; }
+		}
+		public List<string> SHA512s
+		{
+			get { return _sha512s; }
+			set { _sha512s = value; }
+		}
+		public ItemStatus ItemStatuses
+		{
+			get { return _itemStatuses; }
+			set { _itemStatuses = value; }
+		}
+		public MachineType MachineTypes
+		{
+			get { return _machineTypes; }
+			set { _machineTypes = value; }
+		}
+
+		#endregion
+
+		#region Negative
+
+		public List<string> NotGameNames
+		{
+			get { return _notGameNames; }
+			set { _notGameNames = value; }
+		}
+		public List<string> NotRomNames
+		{
+			get { return _notRomNames; }
+			set { _notRomNames = value; }
+		}
+		public List<string> NotRomTypes
+		{
+			get { return _notRomTypes; }
+			set { _notRomTypes = value; }
+		}
+		public List<string> NotCRCs
+		{
+			get { return _notCrcs; }
+			set { _notCrcs = value; }
+		}
+		public List<string> NotMD5s
+		{
+			get { return _notMd5s; }
+			set { _notMd5s = value; }
+		}
+		public List<string> NotSHA1s
+		{
+			get { return _notSha1s; }
+			set { _notSha1s = value; }
+		}
+		public List<string> NotSHA256s
+		{
+			get { return _notSha256s; }
+			set { _notSha256s = value; }
+		}
+		public List<string> NotSHA384s
+		{
+			get { return _notSha384s; }
+			set { _notSha384s = value; }
+		}
+		public List<string> NotSHA512s
+		{
+			get { return _notSha512s; }
+			set { _notSha512s = value; }
+		}
+		public ItemStatus NotItemStatuses
+		{
+			get { return _itemNotStatuses; }
+			set { _itemNotStatuses = value; }
+		}
+		public MachineType NotMachineTypes
+		{
+			get { return _machineNotTypes; }
+			set { _machineNotTypes = value; }
+		}
+
+		#endregion
+
+		#region Neutral
+
+		public long SizeGreaterThanOrEqual
+		{
+			get { return _sizeGreaterThanOrEqual; }
+			set { _sizeGreaterThanOrEqual = value; }
+		}
+		public long SizeLessThanOrEqual
+		{
+			get { return _sizeLessThanOrEqual; }
+			set { _sizeLessThanOrEqual = value; }
+		}
+		public long SizeEqualTo
+		{
+			get { return _sizeEqualTo; }
+			set { _sizeEqualTo = value; }
+		}
+		public bool IncludeOfInGame
+		{
+			get { return _includeOfInGame; }
+			set { _includeOfInGame = value; }
+		}
+		public bool? Runnable
+		{
+			get { return _runnable; }
+			set { _runnable = value; }
+		}
+
+		#endregion
+
+		#endregion // Pubically facing variables
+
+		#region Constructors
+
+		/// <summary>
+		/// Create an empty Filter object
+		/// </summary>
+		public Filter()
+		{
+			// Positive
+			_gameNames = new List<string>();
+			_romNames = new List<string>();
+			_romTypes = new List<string>();
+			_crcs = new List<string>();
+			_md5s = new List<string>();
+			_sha1s = new List<string>();
+			_sha256s = new List<string>();
+			_sha384s = new List<string>();
+			_sha512s = new List<string>();
+			_itemStatuses = ItemStatus.NULL;
+			_machineTypes = MachineType.NULL;
+
+			// Negative
+			_notGameNames = new List<string>();
+			_notRomNames = new List<string>();
+			_notRomTypes = new List<string>();
+			_notCrcs = new List<string>();
+			_notMd5s = new List<string>();
+			_notSha1s = new List<string>();
+			_notSha256s = new List<string>();
+			_notSha384s = new List<string>();
+			_notSha512s = new List<string>();
+			_itemNotStatuses = ItemStatus.NULL;
+			_machineNotTypes = MachineType.NULL;
+
+			// Neutral
+			_sizeGreaterThanOrEqual = -1;
+			_sizeLessThanOrEqual = -1;
+			_sizeEqualTo = -1;
+			_includeOfInGame = false;
+			_runnable = null;
+		}
+
+		#endregion
+
+		#region Instance methods
+
+		/// <summary>
+		/// Check to see if a DatItem passes the filter
+		/// </summary>
+		/// <param name="item">DatItem to check</param>
+		/// <returns>True if the file passed the filter, false otherwise</returns>
+		public bool ItemPasses(DatItem item)
+		{
+			// If the item is null, we automatically fail it
+			if (item == null)
+			{
+				return false;
+			}
+
+			// Filter on machine type
+			if (_machineTypes != MachineType.NULL && (item.MachineType & _machineTypes) == 0)
+			{
+				return false;
+			}
+			if (_machineNotTypes != MachineType.NULL && (item.MachineType & _machineNotTypes) != 0)
+			{
+				return false;
+			}
+
+			// Filter on machine runability
+			if (_runnable != null && item.Runnable != _runnable)
+			{
+				return false;
+			}
+
+			// Take care of Rom and Disk specific differences
+			if (item.Type == ItemType.Rom)
+			{
+				Rom rom = (Rom)item;
+
+				// Filter on status
+				if (_itemStatuses != ItemStatus.NULL && (rom.ItemStatus & _itemStatuses) == 0)
+				{
+					return false;
+				}
+				if (_itemNotStatuses != ItemStatus.NULL && (rom.ItemStatus & _itemNotStatuses) != 0)
+				{
+					return false;
+				}
+
+				// Filter on rom size
+				if (_sizeEqualTo != -1 && rom.Size != _sizeEqualTo)
+				{
+					return false;
+				}
+				else
+				{
+					if (_sizeGreaterThanOrEqual != -1 && rom.Size < _sizeGreaterThanOrEqual)
+					{
+						return false;
+					}
+					if (_sizeLessThanOrEqual != -1 && rom.Size > _sizeLessThanOrEqual)
+					{
+						return false;
+					}
+				}
+
+				// Filter on CRC
+				if (_crcs.Count > 0)
+				{
+					// If the CRC isn't in the list, return false
+					if (!FindValueInList(_crcs, rom.CRC))
+					{
+						return false;
+					}
+				}
+				if (_notCrcs.Count > 0)
+				{
+					// If the CRC is in the list, return false
+					if (FindValueInList(_notCrcs, rom.CRC))
+					{
+						return false;
+					}
+				}
+
+				// Filter on MD5
+				if (_md5s.Count > 0)
+				{
+					// If the MD5 isn't in the list, return false
+					if (!FindValueInList(_md5s, rom.MD5))
+					{
+						return false;
+					}
+				}
+				if (_notMd5s.Count > 0)
+				{
+					// If the MD5 is in the list, return false
+					if (FindValueInList(_notMd5s, rom.MD5))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA1
+				if (_sha1s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha1s, rom.SHA1))
+					{
+						return false;
+					}
+				}
+				if (_notSha1s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha1s, rom.SHA1))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA256
+				if (_sha256s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha256s, rom.SHA256))
+					{
+						return false;
+					}
+				}
+				if (_notSha256s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha256s, rom.SHA256))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA384
+				if (_sha384s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha384s, rom.SHA384))
+					{
+						return false;
+					}
+				}
+				if (_notSha384s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha384s, rom.SHA384))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA512
+				if (_sha512s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha512s, rom.SHA512))
+					{
+						return false;
+					}
+				}
+				if (_notSha512s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha512s, rom.SHA512))
+					{
+						return false;
+					}
+				}
+			}
+			else if (item.Type == ItemType.Disk)
+			{
+				Disk rom = (Disk)item;
+
+				// Filter on status
+				if (_itemStatuses != ItemStatus.NULL && (rom.ItemStatus & _itemStatuses) == 0)
+				{
+					return false;
+				}
+				if (_itemNotStatuses != ItemStatus.NULL && (rom.ItemStatus & _itemNotStatuses) != 0)
+				{
+					return false;
+				}
+
+				// Filter on MD5
+				if (_md5s.Count > 0)
+				{
+					// If the MD5 isn't in the list, return false
+					if (!FindValueInList(_md5s, rom.MD5))
+					{
+						return false;
+					}
+				}
+				if (_notMd5s.Count > 0)
+				{
+					// If the MD5 is in the list, return false
+					if (FindValueInList(_notMd5s, rom.MD5))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA1
+				if (_sha1s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha1s, rom.SHA1))
+					{
+						return false;
+					}
+				}
+				if (_notSha1s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha1s, rom.SHA1))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA256
+				if (_sha256s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha256s, rom.SHA256))
+					{
+						return false;
+					}
+				}
+				if (_notSha256s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha256s, rom.SHA256))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA384
+				if (_sha384s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha384s, rom.SHA384))
+					{
+						return false;
+					}
+				}
+				if (_notSha384s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha384s, rom.SHA384))
+					{
+						return false;
+					}
+				}
+
+				// Filter on SHA512
+				if (_sha512s.Count > 0)
+				{
+					// If the SHA-1 isn't in the list, return false
+					if (!FindValueInList(_sha512s, rom.SHA512))
+					{
+						return false;
+					}
+				}
+				if (_notSha512s.Count > 0)
+				{
+					// If the SHA-1 is in the list, return false
+					if (FindValueInList(_notSha512s, rom.SHA512))
+					{
+						return false;
+					}
+				}
+			}
+
+			// Filter on game name
+			if (_gameNames.Count > 0)
+			{
+				bool found = FindValueInList(_gameNames, item.MachineName);
+
+				// If we are checking CloneOf and RomOf, add them in as well
+				if (_includeOfInGame)
+				{
+					found |= FindValueInList(_gameNames, item.CloneOf);
+					found |= FindValueInList(_gameNames, item.RomOf);
+				}
+
+				// If the game name was not found in the list, return false
+				if (!found)
+				{
+					return false;
+				}
+			}
+			if (_notGameNames.Count > 0)
+			{
+				bool found = FindValueInList(_gameNames, item.MachineName);
+
+				// If we are checking CloneOf and RomOf, add them in as well
+				if (_includeOfInGame)
+				{
+					found |= FindValueInList(_gameNames, item.CloneOf);
+					found |= FindValueInList(_gameNames, item.RomOf);
+				}
+
+				// If the game name was found in the list, return false
+				if (found)
+				{
+					return false;
+				}
+			}
+
+			// Filter on rom name
+			if (_romNames.Count > 0)
+			{
+				// If the rom name was not found in the list, return false
+				if (!FindValueInList(_romNames, item.Name))
+				{
+					return false;
+				}
+			}
+			if (_notRomNames.Count > 0)
+			{
+				// If the rom name was found in the list, return false
+				if (FindValueInList(_notRomNames, item.Name))
+				{
+					return false;
+				}
+			}
+
+			// Filter on rom type
+			if (_romTypes.Count == 0 && _notRomTypes.Count == 0 && item.Type != ItemType.Rom && item.Type != ItemType.Disk)
+			{
+				return false;
+			}
+			if (_romTypes.Count > 0)
+			{
+				// If the rom type was not found in the list, return false
+				if (!FindValueInList(_romTypes, item.Type.ToString()))
+				{
+					return false;
+				}
+			}
+			if (_notRomTypes.Count > 0)
+			{
+				// If the rom type was found in the list, return false
+				if (FindValueInList(_notRomTypes, item.Type.ToString()))
+				{
+					return false;
+				}
+			}
+
+			return true;
+		}
+
+		/// <summary>
+		/// Generic code to check if a specific value is in the list given
+		/// </summary>
+		/// <param name="haystack">List to search for the value in</param>
+		/// <param name="needle">Value to search the list for</param>
+		/// <returns>True if the value could be found, false otherwise</returns>
+		private bool FindValueInList(List<string> haystack, string needle)
+		{
+			bool found = false;
+			foreach (string straw in haystack)
+			{
+				if (!String.IsNullOrEmpty(straw))
+				{
+					string regexStraw = straw;
+
+					// If the straw has no special characters at all, treat it as an exact match
+					if (regexStraw == Regex.Escape(regexStraw))
+					{
+						regexStraw = "^" + regexStraw + "$";
+					}
+
+					// Check if a match is found with the regex
+					found |= Regex.IsMatch(needle, regexStraw, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+				}
+			}
+
+			return found;
+		}
+
+		#endregion
+
+		#region Static methods
+
+		/// <summary>
+		/// Get the machine type from a string
+		/// </summary>
+		/// <param name="gametype">Machine type as a string</param>
+		/// <returns>A machine type based on the input</returns>
+		public static MachineType GetMachineTypeFromString(string gametype)
+		{
+			MachineType machineType = MachineType.NULL;
+			switch (gametype.ToLowerInvariant())
+			{
+				case "none":
+					machineType |= MachineType.None;
+					break;
+				case "bios":
+					machineType |= MachineType.Bios;
+					break;
+				case "dev":
+				case "device":
+					machineType |= MachineType.Device;
+					break;
+				case "mech":
+				case "mechanical":
+					machineType |= MachineType.Mechanical;
+					break;
+				default:
+					Globals.Logger.Warning("{0} is not a valid type", gametype);
+					break;
+			}
+
+			return machineType;
+		}
+
+		/// <summary>
+		/// Get the item status from a string
+		/// </summary>
+		/// <param name="status">Item status as a string</param>
+		/// <returns>An item status based on the input</returns>
+		public static ItemStatus GetStatusFromString(string status)
+		{
+			ItemStatus itemStatus = ItemStatus.NULL;
+			switch (status.ToLowerInvariant())
+			{
+				case "none":
+					itemStatus |= ItemStatus.None;
+					break;
+				case "good":
+					itemStatus |= ItemStatus.Good;
+					break;
+				case "baddump":
+					itemStatus |= ItemStatus.BadDump;
+					break;
+				case "nodump":
+					itemStatus |= ItemStatus.Nodump;
+					break;
+				case "verified":
+					itemStatus |= ItemStatus.Verified;
+					break;
+				default:
+					Globals.Logger.Warning("{0} is not a valid status", status);
+					break;
+			}
+
+			return itemStatus;
+		}
+
+		#endregion
+	}
+}