[ALL] Move object classes to DLL

SabreTools.Helper/Objects/DATFromDir.cs

@@ -0,0 +1,574 @@
+using SabreTools.Helper;
+using SharpCompress.Common;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace SabreTools
+{
+	/// <summary>
+	/// Create a DAT file from a specified file, directory, or set thereof
+	/// </summary>
+	public class DATFromDir
+	{
+		// Path-related variables
+		private string _basePath;
+		private string _tempDir;
+
+		// User specified inputs
+		private List<String> _inputs;
+		private Dat _datdata;
+		private bool _noMD5;
+		private bool _noSHA1;
+		private bool _bare;
+		private bool _archivesAsFiles;
+		private bool _enableGzip;
+		private bool _nowrite;
+
+		// Other required variables
+		private Logger _logger;
+
+		// Public variables
+		public Dat DatData
+		{
+			get { return _datdata; }
+		}
+
+		/// <summary>
+		/// Create a new DATFromDir object
+		/// </summary>
+		/// <param name="inputs">A List of Strings representing the files and folders to be DATted</param>
+		/// <param name="datdata">DatData object representing the requested output DAT</param>
+		/// <param name="noMD5">True if MD5 hashes should be skipped over, false otherwise</param>
+		/// <param name="noSHA1">True if SHA-1 hashes should be skipped over, false otherwise</param>
+		/// <param name="bare">True if the date should be omitted from the DAT, false otherwise</param>
+		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
+		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
+		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
+		/// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param>
+		/// <param name="logger">Logger object for console and file output</param>
+		public DATFromDir(List<String> inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger, bool nowrite = false)
+		{
+			_inputs = inputs;
+			_datdata = datdata;
+			_noMD5 = noMD5;
+			_noSHA1 = noSHA1;
+			_bare = bare;
+			_archivesAsFiles = archivesAsFiles;
+			_enableGzip = enableGzip;
+			_tempDir = tempDir;
+			_logger = logger;
+			_nowrite = nowrite;
+		}
+
+		/// <summary>
+		/// Process the file, folder, or list of some combination into a DAT file
+		/// </summary>
+		/// <returns>True if the DAT could be created, false otherwise</returns>
+		/// <remarks>Try to get the hashing multithreaded (either on a per-hash or per-file level)</remarks>
+		public bool Start()
+		{
+			// Double check to see what it needs to be named
+			_basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : "");
+			_basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : "");
+
+			// If the description is defined but not the name, set the name from the description
+			if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description))
+			{
+				_datdata.Name = _datdata.Description;
+			}
+			
+			// If the name is defined but not the description, set the description from the name
+			else if (!String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+			{
+				_datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+			}
+
+			// If neither the name or description are defined, set them from the automatic values
+			else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+			{
+				if (_inputs.Count > 1)
+				{
+					_datdata.Name = Environment.CurrentDirectory.Split(Path.DirectorySeparatorChar).Last();
+				}
+				else
+				{
+					if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+					{
+						_basePath = _basePath.Substring(0, _basePath.Length - 1);
+					}
+					_datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last();
+				}
+			
+				// If the name is still somehow empty, populate it with defaults
+				_datdata.Name = (String.IsNullOrEmpty(_datdata.Name) ? "Default" : _datdata.Name);
+				_datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+			}
+
+			StreamWriter sw;
+			if (_nowrite)
+			{
+				sw = new StreamWriter(new MemoryStream());
+			}
+			else
+			{
+				// Create and open the output file for writing
+				FileStream fs = File.Create(Style.CreateOutfileName(Environment.CurrentDirectory, _datdata));
+				sw = new StreamWriter(fs, Encoding.UTF8);
+				sw.AutoFlush = true;
+			}
+
+			// Write out the initial file header
+			DatTools.WriteHeader(sw, _datdata, _logger);
+
+			// Loop over each of the found paths, if any
+			string lastparent = null;
+			foreach (string path in _inputs)
+			{
+				// Set local paths and vars
+				_basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar);
+				_basePath = Path.GetFullPath(_basePath);
+
+				// This is where the main loop would go
+				if (File.Exists(_basePath))
+				{
+					lastparent = ProcessPossibleArchive(_basePath, sw, lastparent);
+				}
+				else if (Directory.Exists(_basePath))
+				{
+					_logger.Log("Folder found: " + _basePath);
+
+					// Process the files in the base folder first
+					foreach (string item in Directory.EnumerateFiles(_basePath, "*", SearchOption.TopDirectoryOnly))
+					{
+						lastparent = ProcessPossibleArchive(item, sw, lastparent);
+					}
+
+					// Then process each of the subfolders themselves
+					string basePathBackup = _basePath;
+					foreach (string item in Directory.EnumerateDirectories(_basePath))
+					{
+						if (_datdata.Type != "SuperDAT")
+						{
+							_basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar);
+							_basePath = Path.GetFullPath(_basePath);
+						}
+
+						bool items = false;
+						foreach (string subitem in Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories))
+						{
+							items = true;
+							lastparent = ProcessPossibleArchive(subitem, sw, lastparent);
+						}
+
+						// In romba mode we ignore empty folders completely
+						if (!_datdata.Romba)
+						{
+							// If there were no subitems, add a "blank" game to to the set (if not in Romba mode)
+							if (!items)
+							{
+								string actualroot = item.Remove(0, basePathBackup.Length);
+								Rom rom = new Rom
+								{
+									Name = "null",
+									Machine = new Machine
+									{
+										Name = (_datdata.Type == "SuperDAT" ?
+											(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+												Path.DirectorySeparatorChar.ToString() :
+												"") + actualroot :
+											actualroot),
+									},
+									HashData = new Hash
+									{
+										Size = -1,
+										CRC = "null",
+										MD5 = "null",
+										SHA1 = "null",
+									},
+								};
+
+								string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+								if (_datdata.Files.ContainsKey(key))
+								{
+									_datdata.Files[key].Add(rom);
+								}
+								else
+								{
+									List<Rom> temp = new List<Rom>();
+									temp.Add(rom);
+									_datdata.Files.Add(key, temp);
+								}
+							}
+
+							// Now scour subdirectories for empties and add those as well (if not in Romba mode)
+							foreach (string subdir in Directory.EnumerateDirectories(item, "*", SearchOption.AllDirectories))
+							{
+								if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0)
+								{
+									string actualroot = subdir.Remove(0, basePathBackup.Length);
+									Rom rom = new Rom
+									{
+										Name = "null",
+										Machine = new Machine
+										{
+											Name = (_datdata.Type == "SuperDAT" ?
+												(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+													Path.DirectorySeparatorChar.ToString() :
+													"") + actualroot :
+												actualroot),
+										},
+										HashData = new Hash
+										{
+											Size = -1,
+											CRC = "null",
+											MD5 = "null",
+											SHA1 = "null",
+										},
+									};
+
+									string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+									if (_datdata.Files.ContainsKey(key))
+									{
+										_datdata.Files[key].Add(rom);
+									}
+									else
+									{
+										List<Rom> temp = new List<Rom>();
+										temp.Add(rom);
+										_datdata.Files.Add(key, temp);
+									}
+								}
+							}
+						}
+					}
+					_basePath = basePathBackup;
+				}
+				// If this somehow skips past the original sensors
+				else
+				{
+					_logger.Error(path + " is not a valid input!");
+				}
+			}
+
+			// Now output any empties to the stream (if not in Romba mode)
+			if (!_datdata.Romba)
+			{
+				List<string> keys = _datdata.Files.Keys.ToList();
+				foreach (string key in keys)
+				{
+					List<Rom> roms = _datdata.Files[key];
+					for (int i = 0; i < roms.Count; i++)
+					{
+						Rom rom = roms[i];
+
+						// If we're in a mode that doesn't allow for actual empty folders, add the blank info
+						if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile)
+						{
+							rom.Type = ItemType.Rom;
+							rom.Name = "-";
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+						}
+
+						if (_nowrite)
+						{
+							string inkey = rom.HashData.Size + "-" + rom.HashData.CRC;
+							if (_datdata.Files.ContainsKey(inkey))
+							{
+								_datdata.Files[inkey].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								_datdata.Files.Add(inkey, temp);
+							}
+						}
+						else
+						{
+							// If we have a different game and we're not at the start of the list, output the end of last item
+							int last = 0;
+							if (lastparent != null && lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant())
+							{
+								DatTools.WriteEndGame(sw, rom, new List<string>(), new List<string>(), lastparent, _datdata, 0, out last, _logger);
+							}
+
+							// If we have a new game, output the beginning of the new item
+							if (lastparent == null || lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant())
+							{
+								DatTools.WriteStartGame(sw, rom, new List<string>(), lastparent, _datdata, 0, last, _logger);
+							}
+
+							// Write out the rom data
+							if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile)
+							{
+								DatTools.WriteRomData(sw, rom, lastparent, _datdata, 0, _logger);
+							}
+						}
+
+						lastparent = rom.Machine.Name;
+					}
+				}
+
+				// If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting
+				if (lastparent != null && _datdata.Files.Count == 0)
+				{
+					_datdata.Files.Add("temp", new List<Rom>());
+				}
+			}
+
+			// Now write the final piece and close the output stream
+			DatTools.WriteFooter(sw, _datdata, 0, _logger);
+			sw.Close();
+
+			return true;
+		}
+
+		/// <summary>
+		/// Check a given file for hashes, based on current settings
+		/// </summary>
+		/// <param name="item">Filename of the item to be checked</param>
+		/// <param name="sw">StreamWriter representing the output file</param>
+		/// <param name="lastparent">Name of the last parent rom to make sure that everything is grouped as well as possible</param>
+		/// <returns>New parent to be used</returns>
+		private string ProcessPossibleArchive(string item, StreamWriter sw, string lastparent)
+		{
+			// Define the temporary directory
+			string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir);
+			tempdir += (tempdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? "" : Path.DirectorySeparatorChar.ToString());
+			tempdir += "__temp__" + Path.DirectorySeparatorChar;
+
+			// Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
+			if (_datdata.Romba)
+			{
+				Rom rom = FileTools.GetTorrentGZFileInfo(item, _logger);
+
+				// If the rom is valid, write it out
+				if (rom.Name != null)
+				{
+					int last = 0;
+
+					if (_nowrite)
+					{
+						string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+						if (_datdata.Files.ContainsKey(key))
+						{
+							_datdata.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							_datdata.Files.Add(key, temp);
+						}
+					}
+					else
+					{
+						DatTools.WriteStartGame(sw, rom, new List<string>(), "", _datdata, 0, 0, _logger);
+						DatTools.WriteRomData(sw, rom, "", _datdata, 0, _logger);
+						DatTools.WriteEndGame(sw, rom, new List<string>(), new List<string>(), "", _datdata, 0, out last, _logger);
+					}
+				}
+				else
+				{
+					return string.Empty;
+				}
+
+				_logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+				return rom.Machine.Name;
+			}
+
+			// If both deep hash skip flags are set, do a quickscan
+			if (_noMD5 && _noSHA1)
+			{
+				ArchiveType? type = FileTools.GetCurrentArchiveType(item, _logger);
+
+				// If we have an archive, scan it
+				if (type != null)
+				{
+					List<Rom> extracted = FileTools.GetArchiveFileInfo(item, _logger);
+					foreach (Rom rom in extracted)
+					{
+						lastparent = ProcessFileHelper(item, rom, sw, _basePath,
+							Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) +
+								Path.GetFileNameWithoutExtension(item)
+							), _datdata, lastparent);
+					}
+				}
+				// Otherwise, just get the info on the file itself
+				else if (!Directory.Exists(item) && File.Exists(item))
+				{
+					lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent);
+				}
+			}
+			// Otherwise, attempt to extract the files to the temporary directory
+			else
+			{
+				bool encounteredErrors = FileTools.ExtractArchive(item,
+				tempdir,
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				(!_archivesAsFiles && _enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External),
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				_logger);
+
+				// If the file was an archive and was extracted successfully, check it
+				if (!encounteredErrors)
+				{
+					_logger.Log(Path.GetFileName(item) + " treated like an archive");
+					foreach (string entry in Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories))
+					{
+						string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar);
+						lastparent = ProcessFile(Path.GetFullPath(entry), sw, Path.GetFullPath(tempdir),
+							(String.IsNullOrEmpty(tempbasepath)
+								? ""
+								: (tempbasepath.Length < _basePath.Length
+									? tempbasepath
+									: tempbasepath.Remove(0, _basePath.Length))) +
+							Path.GetFileNameWithoutExtension(item), _datdata, lastparent);
+					}
+
+					// Clear the temp directory
+					if (Directory.Exists(tempdir))
+					{
+						FileTools.CleanDirectory(tempdir);
+					}
+				}
+				// Otherwise, just get the info on the file itself
+				else if (!Directory.Exists(item) && File.Exists(item))
+				{
+					lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent);
+				}
+			}
+
+			return lastparent;
+		}
+
+		/// <summary>
+		/// Process a single file as a file
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="sw">StreamWriter representing the output file</param>
+		/// <param name="basepath">Path the represents the parent directory</param>
+		/// <param name="parent">Parent game to be used</param>
+		/// <param name="datdata">DatData object with output information</param>
+		/// <param name="lastparent">Last known parent game name</param>
+		/// <returns>New last known parent game name</returns>
+		private string ProcessFile(string item, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent)
+		{
+			_logger.Log(Path.GetFileName(item) + " treated like a file");
+			Rom rom = FileTools.GetSingleFileInfo(item, _noMD5, _noSHA1);
+
+			return ProcessFileHelper(item, rom, sw, basepath, parent, datdata, lastparent);
+		}
+
+		/// <summary>
+		/// Process a single file as a file (with found Rom data)
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="rom">Rom data to be used to write to file</param>
+		/// <param name="sw">StreamWriter representing the output file</param>
+		/// <param name="basepath">Path the represents the parent directory</param>
+		/// <param name="parent">Parent game to be used</param>
+		/// <param name="datdata">DatData object with output information</param>
+		/// <param name="lastparent">Last known parent game name</param>
+		/// <returns>New last known parent game name</returns>
+		private string ProcessFileHelper(string item, Rom rom, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent)
+		{
+			try
+			{
+				if (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					basepath = basepath.Substring(0, basepath.Length - 1);
+				}
+
+				string actualroot = (item == basepath ? item.Split(Path.DirectorySeparatorChar).Last() : item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0]);
+				if (parent == "")
+				{
+					actualroot = (actualroot == "" && datdata.Type != "SuperDAT" ? basepath.Split(Path.DirectorySeparatorChar).Last() : actualroot);
+				}
+				string actualitem = (item == basepath ? item : item.Remove(0, basepath.Length + 1));
+
+				// If we're in SuperDAT mode, make sure the added item is by itself
+				if (datdata.Type == "SuperDAT")
+				{
+					actualroot += (actualroot != "" ? Path.DirectorySeparatorChar.ToString() : "") +
+						(parent != "" ? parent + Path.DirectorySeparatorChar : "") +
+						Path.GetDirectoryName(actualitem);
+					actualroot = actualroot.TrimEnd(Path.DirectorySeparatorChar);
+					actualitem = Path.GetFileName(actualitem);
+				}
+				else if (parent != "")
+				{
+					actualroot = parent.TrimEnd(Path.DirectorySeparatorChar);
+				}
+
+				// Drag and drop is funny
+				if (actualitem == Path.GetFullPath(actualitem))
+				{
+					actualitem = Path.GetFileName(actualitem);
+				}
+
+				_logger.Log("Actual item added: " + actualitem);
+
+				// Update rom information
+				rom.Machine = new Machine
+				{
+					Name = (datdata.Type == "SuperDAT" ?
+						(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+							Path.DirectorySeparatorChar.ToString() :
+							"") + actualroot :
+						actualroot),
+				};
+				rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString());
+				rom.Name = actualitem;
+
+				if (_nowrite)
+				{
+					string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+					if (_datdata.Files.ContainsKey(key))
+					{
+						_datdata.Files[key].Add(rom);
+					}
+					else
+					{
+						List<Rom> temp = new List<Rom>();
+						temp.Add(rom);
+						_datdata.Files.Add(key, temp);
+					}
+				}
+				else
+				{
+					// If we have a different game and we're not at the start of the list, output the end of last item
+					int last = 0;
+					if (lastparent != null && lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant())
+					{
+						DatTools.WriteEndGame(sw, rom, new List<string>(), new List<string>(), lastparent, datdata, 0, out last, _logger);
+					}
+
+					// If we have a new game, output the beginning of the new item
+					if (lastparent == null || lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant())
+					{
+						DatTools.WriteStartGame(sw, rom, new List<string>(), lastparent, datdata, 0, last, _logger);
+					}
+
+					// Write out the rom data
+					DatTools.WriteRomData(sw, rom, lastparent, datdata, 0, _logger);
+				}
+				_logger.User("File added: " + actualitem + Environment.NewLine);
+
+				return rom.Machine.Name;
+			}
+			catch (IOException ex)
+			{
+				_logger.Error(ex.ToString());
+				return null;
+			}
+		}
+	}
+}

SabreTools.Helper/Objects/DATFromDirParallel.cs

@@ -0,0 +1,494 @@
+using SabreTools.Helper;
+using SharpCompress.Common;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace SabreTools
+{
+	/// <summary>
+	/// Create a DAT file from a specified file, directory, or set thereof
+	/// </summary>
+	public class DATFromDirParallel
+	{
+		// Path-related variables
+		private string _basePath;
+		private string _tempDir;
+
+		// User specified inputs
+		private List<String> _inputs;
+		private Dat _datdata;
+		private bool _noMD5;
+		private bool _noSHA1;
+		private bool _bare;
+		private bool _archivesAsFiles;
+		private bool _enableGzip;
+
+		// Other required variables
+		private Logger _logger;
+
+		// Public variables
+		public Dat DatData
+		{
+			get { return _datdata; }
+		}
+
+		/// <summary>
+		/// Create a new DATFromDir object
+		/// </summary>
+		/// <param name="inputs">A List of Strings representing the files and folders to be DATted</param>
+		/// <param name="datdata">DatData object representing the requested output DAT</param>
+		/// <param name="noMD5">True if MD5 hashes should be skipped over, false otherwise</param>
+		/// <param name="noSHA1">True if SHA-1 hashes should be skipped over, false otherwise</param>
+		/// <param name="bare">True if the date should be omitted from the DAT, false otherwise</param>
+		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
+		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
+		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
+		/// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param>
+		/// <param name="logger">Logger object for console and file output</param>
+		public DATFromDirParallel(List<String> inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger)
+		{
+			_inputs = inputs;
+			_datdata = datdata;
+			_noMD5 = noMD5;
+			_noSHA1 = noSHA1;
+			_bare = bare;
+			_archivesAsFiles = archivesAsFiles;
+			_enableGzip = enableGzip;
+			_tempDir = tempDir;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Process the file, folder, or list of some combination into a DAT file
+		/// </summary>
+		/// <returns>True if the DAT could be created, false otherwise</returns>
+		/// <remarks>Try to get the hashing multithreaded (either on a per-hash or per-file level)</remarks>
+		public bool Start()
+		{
+			// Double check to see what it needs to be named
+			_basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : "");
+			_basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : "");
+
+			// If the description is defined but not the name, set the name from the description
+			if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description))
+			{
+				_datdata.Name = _datdata.Description;
+			}
+
+			// If the name is defined but not the description, set the description from the name
+			else if (!String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+			{
+				_datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+			}
+
+			// If neither the name or description are defined, set them from the automatic values
+			else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+			{
+				if (_inputs.Count > 1)
+				{
+					_datdata.Name = Environment.CurrentDirectory.Split(Path.DirectorySeparatorChar).Last();
+				}
+				else
+				{
+					if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+					{
+						_basePath = _basePath.Substring(0, _basePath.Length - 1);
+					}
+					_datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last();
+				}
+
+				// If the name is still somehow empty, populate it with defaults
+				_datdata.Name = (String.IsNullOrEmpty(_datdata.Name) ? "Default" : _datdata.Name);
+				_datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+			}
+
+			// Loop over each of the found paths, if any
+			string lastparent = null;
+			foreach (string path in _inputs)
+			{
+				// Set local paths and vars
+				_basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar);
+				_basePath = Path.GetFullPath(_basePath);
+
+				// This is where the main loop would go
+				if (File.Exists(_basePath))
+				{
+					lastparent = ProcessPossibleArchive(_basePath, lastparent);
+				}
+				else if (Directory.Exists(_basePath))
+				{
+					_logger.Log("Folder found: " + _basePath);
+
+					// Process the files in the base folder first
+					Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.TopDirectoryOnly), item =>
+					{
+						lastparent = ProcessPossibleArchive(item, lastparent);
+					});
+
+					// Then process each of the subfolders themselves
+					string basePathBackup = _basePath;
+					foreach (string item in Directory.EnumerateDirectories(_basePath))
+					{
+						if (_datdata.Type != "SuperDAT")
+						{
+							_basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar);
+							_basePath = Path.GetFullPath(_basePath);
+						}
+
+						bool items = false;
+						Parallel.ForEach(Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories), subitem =>
+						{
+							items = true;
+							lastparent = ProcessPossibleArchive(subitem, lastparent);
+						});
+
+						// In romba mode we ignore empty folders completely
+						if (!_datdata.Romba)
+						{
+							// If there were no subitems, add a "blank" game to to the set (if not in Romba mode)
+							if (!items)
+							{
+								string actualroot = item.Remove(0, basePathBackup.Length);
+								Rom rom = new Rom
+								{
+									Name = "null",
+									Machine = new Machine
+									{
+										Name = (_datdata.Type == "SuperDAT" ?
+											(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+												Path.DirectorySeparatorChar.ToString() :
+												"") + actualroot :
+											actualroot),
+									},
+									HashData = new Hash
+									{
+										Size = -1,
+										CRC = "null",
+										MD5 = "null",
+										SHA1 = "null",
+									},
+								};
+
+								string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+								if (_datdata.Files.ContainsKey(key))
+								{
+									_datdata.Files[key].Add(rom);
+								}
+								else
+								{
+									List<Rom> temp = new List<Rom>();
+									temp.Add(rom);
+									_datdata.Files.Add(key, temp);
+								}
+							}
+
+							// Now scour subdirectories for empties and add those as well (if not in Romba mode)
+							foreach (string subdir in Directory.EnumerateDirectories(item, "*", SearchOption.AllDirectories))
+							{
+								if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0)
+								{
+									string actualroot = subdir.Remove(0, basePathBackup.Length);
+									Rom rom = new Rom
+									{
+										Name = "null",
+										Machine = new Machine
+										{
+											Name = (_datdata.Type == "SuperDAT" ?
+												(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+													Path.DirectorySeparatorChar.ToString() :
+													"") + actualroot :
+												actualroot),
+										},
+										HashData = new Hash
+										{
+											Size = -1,
+											CRC = "null",
+											MD5 = "null",
+											SHA1 = "null",
+										},
+									};
+
+									string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+									if (_datdata.Files.ContainsKey(key))
+									{
+										_datdata.Files[key].Add(rom);
+									}
+									else
+									{
+										List<Rom> temp = new List<Rom>();
+										temp.Add(rom);
+										_datdata.Files.Add(key, temp);
+									}
+								}
+							}
+						}
+					}
+					_basePath = basePathBackup;
+				}
+				// If this somehow skips past the original sensors
+				else
+				{
+					_logger.Error(path + " is not a valid input!");
+				}
+			}
+
+			// Now output any empties to the stream (if not in Romba mode)
+			if (!_datdata.Romba)
+			{
+				List<string> keys = _datdata.Files.Keys.ToList();
+				foreach (string key in keys)
+				{
+					List<Rom> roms = _datdata.Files[key];
+					for (int i = 0; i < roms.Count; i++)
+					{
+						Rom rom = roms[i];
+
+						// If we're in a mode that doesn't allow for actual empty folders, add the blank info
+						if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile)
+						{
+							rom.Type = ItemType.Rom;
+							rom.Name = "-";
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+						}
+
+						string inkey = rom.HashData.Size + "-" + rom.HashData.CRC;
+						if (_datdata.Files.ContainsKey(inkey))
+						{
+							_datdata.Files[inkey].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							_datdata.Files.Add(inkey, temp);
+						}
+
+						lastparent = rom.Machine.Name;
+					}
+				}
+
+				// If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting
+				if (lastparent != null && _datdata.Files.Count == 0)
+				{
+					_datdata.Files.Add("temp", new List<Rom>());
+				}
+			}
+
+			return true;
+		}
+
+		/// <summary>
+		/// Check a given file for hashes, based on current settings
+		/// </summary>
+		/// <param name="item">Filename of the item to be checked</param>
+		/// <param name="lastparent">Name of the last parent rom to make sure that everything is grouped as well as possible</param>
+		/// <returns>New parent to be used</returns>
+		private string ProcessPossibleArchive(string item, string lastparent)
+		{
+			// Define the temporary directory
+			string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir);
+			tempdir += (tempdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? "" : Path.DirectorySeparatorChar.ToString());
+			tempdir += "__temp__" + Path.DirectorySeparatorChar;
+
+			// Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
+			if (_datdata.Romba)
+			{
+				Rom rom = FileTools.GetTorrentGZFileInfo(item, _logger);
+
+				// If the rom is valid, write it out
+				if (rom.Name != null)
+				{
+					string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+					if (_datdata.Files.ContainsKey(key))
+					{
+						_datdata.Files[key].Add(rom);
+					}
+					else
+					{
+						List<Rom> temp = new List<Rom>();
+						temp.Add(rom);
+						_datdata.Files.Add(key, temp);
+					}
+				}
+				else
+				{
+					return string.Empty;
+				}
+
+				_logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+				return rom.Machine.Name;
+			}
+
+			// If both deep hash skip flags are set, do a quickscan
+			if (_noMD5 && _noSHA1)
+			{
+				ArchiveType? type = FileTools.GetCurrentArchiveType(item, _logger);
+
+				// If we have an archive, scan it
+				if (type != null)
+				{
+					List<Rom> extracted = FileTools.GetArchiveFileInfo(item, _logger);
+					foreach (Rom rom in extracted)
+					{
+						lastparent = ProcessFileHelper(item, rom, _basePath,
+							Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) +
+								Path.GetFileNameWithoutExtension(item)
+							), _datdata, lastparent);
+					}
+				}
+				// Otherwise, just get the info on the file itself
+				else if (!Directory.Exists(item) && File.Exists(item))
+				{
+					lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent);
+				}
+			}
+			// Otherwise, attempt to extract the files to the temporary directory
+			else
+			{
+				bool encounteredErrors = FileTools.ExtractArchive(item,
+				tempdir,
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				(!_archivesAsFiles && _enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External),
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				(_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+				_logger);
+
+				// If the file was an archive and was extracted successfully, check it
+				if (!encounteredErrors)
+				{
+					_logger.Log(Path.GetFileName(item) + " treated like an archive");
+					Parallel.ForEach(Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories), entry =>
+					{
+						string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar);
+						lastparent = ProcessFile(Path.GetFullPath(entry), Path.GetFullPath(tempdir),
+							(String.IsNullOrEmpty(tempbasepath)
+								? ""
+								: (tempbasepath.Length < _basePath.Length
+									? tempbasepath
+									: tempbasepath.Remove(0, _basePath.Length))) +
+							Path.GetFileNameWithoutExtension(item), _datdata, lastparent);
+					});
+
+					// Clear the temp directory
+					if (Directory.Exists(tempdir))
+					{
+						FileTools.CleanDirectory(tempdir);
+					}
+				}
+				// Otherwise, just get the info on the file itself
+				else if (!Directory.Exists(item) && File.Exists(item))
+				{
+					lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent);
+				}
+			}
+
+			return lastparent;
+		}
+
+		/// <summary>
+		/// Process a single file as a file
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="basepath">Path the represents the parent directory</param>
+		/// <param name="parent">Parent game to be used</param>
+		/// <param name="datdata">DatData object with output information</param>
+		/// <param name="lastparent">Last known parent game name</param>
+		/// <returns>New last known parent game name</returns>
+		private string ProcessFile(string item, string basepath, string parent, Dat datdata, string lastparent)
+		{
+			_logger.Log(Path.GetFileName(item) + " treated like a file");
+			Rom rom = FileTools.GetSingleFileInfo(item, _noMD5, _noSHA1);
+
+			return ProcessFileHelper(item, rom, basepath, parent, datdata, lastparent);
+		}
+
+		/// <summary>
+		/// Process a single file as a file (with found Rom data)
+		/// </summary>
+		/// <param name="item">File to be added</param>
+		/// <param name="rom">Rom data to be used to write to file</param>
+		/// <param name="basepath">Path the represents the parent directory</param>
+		/// <param name="parent">Parent game to be used</param>
+		/// <param name="datdata">DatData object with output information</param>
+		/// <param name="lastparent">Last known parent game name</param>
+		/// <returns>New last known parent game name</returns>
+		private string ProcessFileHelper(string item, Rom rom, string basepath, string parent, Dat datdata, string lastparent)
+		{
+			try
+			{
+				if (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+				{
+					basepath = basepath.Substring(0, basepath.Length - 1);
+				}
+
+				string actualroot = (item == basepath ? item.Split(Path.DirectorySeparatorChar).Last() : item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0]);
+				if (parent == "")
+				{
+					actualroot = (actualroot == "" && datdata.Type != "SuperDAT" ? basepath.Split(Path.DirectorySeparatorChar).Last() : actualroot);
+				}
+				string actualitem = (item == basepath ? item : item.Remove(0, basepath.Length + 1));
+
+				// If we're in SuperDAT mode, make sure the added item is by itself
+				if (datdata.Type == "SuperDAT")
+				{
+					actualroot += (actualroot != "" ? Path.DirectorySeparatorChar.ToString() : "") +
+						(parent != "" ? parent + Path.DirectorySeparatorChar : "") +
+						Path.GetDirectoryName(actualitem);
+					actualroot = actualroot.TrimEnd(Path.DirectorySeparatorChar);
+					actualitem = Path.GetFileName(actualitem);
+				}
+				else if (parent != "")
+				{
+					actualroot = parent.TrimEnd(Path.DirectorySeparatorChar);
+				}
+
+				// Drag and drop is funny
+				if (actualitem == Path.GetFullPath(actualitem))
+				{
+					actualitem = Path.GetFileName(actualitem);
+				}
+
+				_logger.Log("Actual item added: " + actualitem);
+
+				// Update rom information
+				rom.Machine = new Machine
+				{
+					Name = (datdata.Type == "SuperDAT" ?
+						(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+							Path.DirectorySeparatorChar.ToString() :
+							"") + actualroot :
+						actualroot),
+				};
+				rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString());
+				rom.Name = actualitem;
+
+				string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+				if (_datdata.Files.ContainsKey(key))
+				{
+					_datdata.Files[key].Add(rom);
+				}
+				else
+				{
+					List<Rom> temp = new List<Rom>();
+					temp.Add(rom);
+					_datdata.Files.Add(key, temp);
+				}
+				_logger.User("File added: " + actualitem + Environment.NewLine);
+
+				return rom.Machine.Name;
+			}
+			catch (IOException ex)
+			{
+				_logger.Error(ex.ToString());
+				return null;
+			}
+		}
+	}
+}

SabreTools.Helper/Objects/Generate.cs

@@ -0,0 +1,472 @@
+using Mono.Data.Sqlite;
+using SabreTools.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace SabreTools
+{
+	/// <summary>
+	/// Generate a DAT from the data in the database
+	/// </summary>
+	class Generate : IGenerate
+	{
+		// Private instance variables
+		private string _systems;
+		private string _sources;
+		private string _outdir;
+		private string _connectionString;
+		private bool _norename;
+		private bool _old;
+
+		// Private required variables
+		private Logger _logger;
+
+		/// <summary>
+		/// Initialize a Generate object with the given information
+		/// </summary>
+		/// <param name="systems">Comma-separated list of systems to be included in the DAT (blank means all)</param>
+		/// <param name="sources">Comma-separated list of sources to be included in the DAT (blank means all)</param>
+		/// <param name="outdir">The output folder where the generated DAT will be put; blank means the current directory</param>
+		/// <param name="connectionString">Connection string for SQLite</param>
+		/// <param name="logger">Logger object for file or console output</param>
+		/// <param name="norename">True if files should not be renamed with system and/or source in merged mode (default false)</param>
+		/// <param name="old">True if the output file should be in ClrMamePro format (default false)</param>
+		public Generate(string systems, string sources, string outdir, string connectionString, Logger logger, bool norename = false, bool old = false)
+		{
+			_systems = systems;
+			_sources = sources;
+			_connectionString = connectionString;
+			_norename = norename;
+			_old = old;
+			_logger = logger;
+
+			// Take care of special outfolder cases
+			_outdir = (outdir == "" ? Environment.CurrentDirectory + Path.DirectorySeparatorChar :
+				(!outdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? outdir + Path.DirectorySeparatorChar : outdir)
+			);
+			if (_outdir != "" && !Directory.Exists(_outdir))
+			{
+				Directory.CreateDirectory(_outdir);
+			}
+		}
+
+		/// <summary>
+		/// Generate a DAT file that is represented by the data in the Generate object.
+		/// </summary>
+		/// <returns>True if the file could be created, false otherwise</returns>
+		public bool Export()
+		{
+			// Check to see if the source is an import-only. If so, tell the user and exit
+			int id = 0;
+			if (_sources != "" && Int32.TryParse(_sources, out id) && id <= 14)
+			{
+				_logger.Warning("This source (" + id + ") is import-only so a DAT cannot be created. We apologize for the inconvenience.");
+				return false;
+			}
+
+			// Get the system name, if applicable
+			string systemname = "";
+			if (_systems != "")
+			{
+				string query = "SELECT manufacturer, system FROM systems WHERE id in (" + _systems + ")";
+				//string query = "SELECT manufacturer, name FROM system WHERE id in (" + _systems + ")";
+
+				using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+				{
+					dbc.Open();
+					using (SqliteCommand slc = new SqliteCommand(query, dbc))
+					{
+						using (SqliteDataReader sldr = slc.ExecuteReader())
+						{
+							// If there are no games for this combination, return nothing
+							if (!sldr.HasRows)
+							{
+								_logger.Error("No system could be found with id in \"" + _systems + "\". Please check and try again.");
+								return false;
+							}
+
+							// Retrieve and build the system name from all retrieved
+							int tempsize = 0;
+							while (sldr.Read() && tempsize < 3)
+							{
+								systemname += (tempsize == 0 ?
+									sldr.GetString(0) + " - " + sldr.GetString(1) :
+									"; " + sldr.GetString(0) + " - " + sldr.GetString(1));
+								tempsize++;
+							}
+
+							// If there are more than 3 systems, just put "etc." on the end
+							if (sldr.Read())
+							{
+								systemname += "; etc.";
+							}
+						}
+					}
+				}
+			}
+			else
+			{
+				systemname = "ALL";
+			}
+
+			string sourcename = "";
+			if (_sources != "")
+			{
+				string query = "SELECT name FROM sources WHERE id in (" + _sources + ")";
+				//string query = "SELECT name FROM source WHERE id in (" + _sources + ")";
+
+				using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+				{
+					dbc.Open();
+					using (SqliteCommand slc = new SqliteCommand(query, dbc))
+					{
+						using (SqliteDataReader sldr = slc.ExecuteReader())
+						{
+							// If there are no games for this combination, return nothing
+							if (!sldr.HasRows)
+							{
+								_logger.Error("No source could be found with id in \"" + _sources + "\". Please check and try again.");
+								return false;
+							}
+
+							// Retrieve and build the source name from all retrieved
+							int tempsize = 0;
+							while (sldr.Read() && tempsize < 3)
+							{
+								sourcename += (tempsize == 0 ? sldr.GetString(0) : "; " + sldr.GetString(0));
+								tempsize++;
+							}
+
+							// If there are more than 3 systems, just put "etc." on the end
+							if (sldr.Read())
+							{
+								sourcename += "; etc.";
+							}
+						}
+					}
+				}
+			}
+			else
+			{
+				sourcename = "Merged";
+			}
+
+			// Retrieve the list of processed roms
+			Dictionary<string, List<Rom>> dict = ProcessRoms();
+
+			// If the output is null, nothing was found so return false
+			if (dict.Count == 0)
+			{
+				return false;
+			}
+
+			// Create a name for the file based on the retrieved information
+			string version = DateTime.Now.ToString("yyyyMMddHHmmss");
+			string intname = systemname + " (" + sourcename + ")";
+			string datname = systemname + " (" + sourcename + " " + version + ")";
+
+			Dat datdata = new Dat
+			{
+				Name = intname,
+				Description = datname,
+				Version = version,
+				Date = version,
+				Category = "The Wizard of DATz",
+				Author = "The Wizard of DATz",
+				ForcePacking = ForcePacking.None,
+				OutputFormat = (_old ? OutputFormat.ClrMamePro : OutputFormat.Xml),
+				Files = dict,
+			};
+
+			return DatTools.WriteDatfile(datdata, _outdir, _logger);
+		}
+
+		/// <summary>
+		/// Preprocess the rom data that is to be included in the outputted DAT
+		/// </summary>
+		/// <returns>A List of Rom objects containing all information about the files</returns>
+		public Dictionary<string, List<Rom>> ProcessRoms()
+		{
+			Dictionary<string, List<Rom>> roms = new Dictionary<string, List<Rom>>();
+
+			// Check if we have listed sources or systems
+			bool sysmerged = (_systems == "" || _systems.Split(',').Length > 1);
+			bool srcmerged = (_sources == "" || _sources.Split(',').Length > 1);
+			bool merged = sysmerged || srcmerged;
+
+			// BEGIN COMMENT
+			string query = @"
+SELECT DISTINCT systems.manufacturer AS manufacturer, systems.system AS system, systems.id AS systemid,
+	sources.name AS source, sources.url AS url, sources.id AS sourceid,
+	games.name AS game, files.name AS name, files.type AS type,
+	checksums.size AS size, checksums.crc AS crc, checksums.md5 AS md5, checksums.sha1 AS sha1,
+	files.lastupdated AS lastupdated
+FROM systems
+JOIN games
+	ON systems.id=games.system
+JOIN sources
+	ON games.source=sources.id
+JOIN files
+	ON games.id=files.setid
+JOIN checksums
+	ON files.id=checksums.file" +
+	(_systems != "" || _sources != "" ? "\nWHERE" : "") +
+	(_sources != "" ? " sources.id in (" + _sources + ")" : "") +
+	(_systems != "" && _sources != "" ? " AND" : "") +
+	(_systems != "" ? " systems.id in (" + _systems + ")" : "") +
+"\nORDER BY " +
+	(merged ? "checksums.size, checksums.crc, systems.id, sources.id, files.lastupdated DESC, checksums.md5, checksums.sha1"
+			: "systems.id, sources.id, games.name, files.name");
+
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						// If there are no games for this combination, return nothing
+						if (!sldr.HasRows)
+						{
+							_logger.Error("No games could be found with those inputs. Please check and try again.");
+							return null;
+						}
+
+						// Retrieve and process the roms for merging
+						while (sldr.Read())
+						{
+							Rom temp = new Rom
+							{
+								Name = sldr.GetString(7),
+								Type = (sldr.GetString(8) == "disk" ? ItemType.Disk : ItemType.Rom),
+								Machine = new Machine
+								{
+									Name = sldr.GetString(6),
+									Manufacturer = sldr.GetString(0),
+								},
+								Metadata = new SourceMetadata
+								{
+									System = sldr.GetString(1),
+									SystemID = sldr.GetInt32(2),
+									Source = sldr.GetString(3),
+									SourceID = sldr.GetInt32(5),
+								},
+								HashData = new Hash
+								{
+									Size = sldr.GetInt64(9),
+									CRC = sldr.GetString(10),
+									MD5 = sldr.GetString(11),
+									SHA1 = sldr.GetString(12),
+								},
+							};
+
+							// Rename the game associated if it's still valid and we allow renames
+							if (merged && !_norename)
+							{
+								temp.Machine.Name = temp.Machine.Name +
+									(sysmerged ? " [" + temp.Machine.Manufacturer + " - " + temp.Metadata.System + "]" : "") +
+									(srcmerged ? " [" + temp.Metadata.Source + "]" : "");
+							}
+
+							string key = temp.HashData.Size + "-" + temp.HashData.CRC;
+							if (roms.ContainsKey(key))
+							{
+								roms[key].Add(temp);
+							}
+							else
+							{
+								List<Rom> templist = new List<Rom>();
+								templist.Add(temp);
+								roms.Add(key, templist);
+							}
+						}
+					}
+				}
+			}
+
+			// If we're in a merged mode, merge and then resort by the correct parameters
+			if (merged)
+			{
+				foreach (string key in roms.Keys)
+				{
+					roms[key] = RomTools.Merge(roms[key], _logger);
+				}
+			}
+			// END COMMENT
+
+			/*
+			// This block would replace the whole block above between BEGIN COMMENT and END COMMENT
+			string query = @"
+SELECT hash.id AS id, hash.size AS size, hash.crc AS crc, hash.md5 AS md5, hash.sha1 AS sha1,
+	a.key AS key, a.value AS value,
+	source.id, source.name, source.url,
+	system.id, system.manufacturer, system.name
+FROM hash
+JOIN hashdata a
+	ON hash.id=a.hashid
+JOIN hashdata b
+	ON a.hashid=b.hashid
+JOIN gamesystem
+	ON b.value=gamesystem.game
+JOIN gamesource
+	ON b.value=gamesource.game
+JOIN system
+	ON gamesystem.systemid=system.id
+JOIN source
+	ON gamesource.sourceid=source.id" + 
+(_systems != "" || _sources != "" ? "\nWHERE" : "") +
+	(_sources != "" ? " source.id in (" + _sources + ")" : "") +
+	(_systems != "" && _sources != "" ? " AND" : "") +
+	(_systems != "" ? " system.id in (" + _systems + ")" : "") +
+"\nORDER BY hash.id";
+
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						// If there are no games for this combination, return nothing
+						if (!sldr.HasRows)
+						{
+							_logger.Error("No games could be found with those inputs. Please check and try again.");
+							return null;
+						}
+
+						// Retrieve and process the roms for merging
+						int systemid = -1, sourceid = -1;
+						long lastid = -1, size = -1;
+						string name = "", game = "", type = "", manufacturer = "", system = "", source = "", url = "", crc = "", md5 = "", sha1 = "";
+						while (sldr.Read())
+						{
+							// If the hash is different than the last
+							if (lastid != -1 && sldr.GetInt64(0) != lastid)
+							{
+								Rom temp = new Rom
+								{
+									Manufacturer = manufacturer,
+									System = system,
+									SystemID = systemid,
+									Source = source,
+									URL = url,
+									SourceID = sourceid,
+									Game = game,
+									Name = name,
+									Type = type,
+									Size = size,
+									CRC = crc,
+									MD5 = md5,
+									SHA1 = sha1,
+								};
+
+								// Rename the game associated if it's still valid and we allow renames
+								if (merged && !_norename)
+								{
+									temp.Machine = temp.Machine +
+										(sysmerged ? " [" + temp.Manufacturer + " - " + temp.System + "]" : "") +
+										(srcmerged ? " [" + temp.Source + "]" : "");
+								}
+
+								string key = temp.Size + "-" + temp.CRC;
+								if (roms.ContainsKey(key))
+								{
+									roms[key].Add(temp);
+								}
+								else
+								{
+									List<Rom> templist = new List<Rom>();
+									templist.Add(temp);
+									roms.Add(key, templist);
+								}
+
+								// Reset the variables
+								game = "";
+								name = "";
+								type = "";
+							}
+							
+							// Get all of the current ROM information
+							manufacturer = sldr.GetString(11);
+							system = sldr.GetString(12);
+							systemid = sldr.GetInt32(10);
+							source = sldr.GetString(8);
+							url = sldr.GetString(9);
+							sourceid = sldr.GetInt32(7);
+							size = sldr.GetInt64(1);
+							crc = sldr.GetString(2);
+							md5 = sldr.GetString(3);
+							sha1 = sldr.GetString(4);
+
+							switch (sldr.GetString(5))
+							{
+								case "game":
+									game = sldr.GetString(6);
+									break;
+								case "name":
+									name = sldr.GetString(6);
+									break;
+								case "type":
+									type = sldr.GetString(6);
+									break;
+							}
+
+							lastid = sldr.GetInt64(0);
+						}
+					}
+				}
+			}
+
+			// If we're in a merged mode, merge
+			if (merged)
+			{
+				foreach (string key in roms.Keys)
+				{
+					roms[key] = RomManipulation.Merge(roms[key]);
+				}
+			}
+			*/
+
+			/*
+			// THIS CODE SHOULD BE PUT IN WriteToDatFromDict
+
+			// Now check rename within games
+			string lastname = "", lastgame = "";
+			for (int i = 0; i < roms.Count; i++)
+			{
+				Rom rom = roms[i];
+
+				// Now relable any roms that have the same name inside of the same game
+				bool samename = false, samegame = false;
+				if (rom.Name != "")
+				{
+					samename = (lastname == rom.Name);
+				}
+				if (rom.Machine != "")
+				{
+					samegame = (lastgame == rom.Machine);
+				}
+
+				lastname = rom.Name;
+				lastgame = rom.Machine;
+
+				// If the name and set are the same, rename it with whatever is different
+				if (samename && samegame)
+				{
+					rom.Name = Regex.Replace(rom.Name, @"^(.*)(\..*)", "$1(" +
+							(rom.CRC != "" ? rom.CRC :
+									(rom.MD5 != "" ? rom.MD5 :
+											(rom.SHA1 != "" ? rom.SHA1 : "Alt"))) +
+							")$2");
+				}
+
+				// Assign back just in case
+				roms[i] = rom;
+			}
+			*/
+
+			return roms;
+		}
+	}
+}

SabreTools.Helper/Objects/GenerateTwo.cs

@@ -0,0 +1,246 @@
+using Mono.Data.Sqlite;
+using SabreTools.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text.RegularExpressions;
+
+namespace SabreTools
+{
+	public class GenerateTwo : IGenerate
+	{
+		// Private instance variables
+		private string _systemid;
+		private string _sourceid;
+		private string _datroot;
+		private string _outroot;
+		private string _connectionString;
+		private bool _norename;
+		private bool _old;
+
+		// Private required variables
+		private Logger _logger;
+
+		/// <summary>
+		/// Initialize a Generate object with the given information
+		/// </summary>
+		/// <param name="systemid">String representing the system id (blank means all)</param>
+		/// <param name="sourceid">String representing the source id (blank means all) [CURRENTLY UNUSED]</param>
+		/// <param name="datroot">Root directory where all DAT files are held</param>
+		/// <param name="outroot">Root directory where new DAT files are output</param>
+		/// <param name="connectionString">Connection string for SQLite</param>
+		/// <param name="logger">Logger object for file or console output</param>
+		/// <param name="norename">True if files should not be renamed with system and/or source in merged mode (default false)</param>
+		/// <param name="old">True if the output file should be in ClrMamePro format (default false)</param>
+		public GenerateTwo(string systemid, string sourceid, string datroot, string outroot, string connectionString, Logger logger, bool norename = false, bool old = false)
+		{
+			_systemid = systemid;
+			_sourceid = sourceid;
+			_datroot = datroot;
+			_outroot = outroot;
+			_connectionString = connectionString;
+			_logger = logger;
+			_norename = norename;
+			_old = old;
+		}
+
+		/// <summary>
+		/// Generate a DAT file that is represented by the data in the Generate object.
+		/// </summary>
+		/// <returns>True if the file could be created, false otherwise</returns>
+		public bool Export()
+		{
+			string name = "";
+			string path = _datroot;
+
+			// If the System ID isn't set, then we will merge everything
+			if (_systemid != "")
+			{
+				string system = "";
+
+				// First get the name of the system, if possible
+				string query = "SELECT manufacturer, name FROM system WHERE id=" + _systemid;
+				using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+				{
+					dbc.Open();
+
+					using (SqliteCommand slc = new SqliteCommand(query, dbc))
+					{
+						using (SqliteDataReader sldr = slc.ExecuteReader())
+						{
+							if (sldr.Read())
+							{
+								system = sldr.GetString(0) + " - " + sldr.GetString(1);
+							}
+						}
+					}
+				}
+
+				// If we didn't find anything, then return
+				if (system == "")
+				{
+					_logger.Warning("No system could be found with id " + _systemid);
+					return false;
+				}
+
+				name = system.Trim();
+				path = Path.Combine(path, name);
+			}
+			else
+			{
+				name = "ALL";
+			}
+
+			// Get the rest of the info as well
+			string date = DateTime.Now.ToString("yyyyMMddHHmmss");
+			string description = name + " (merged " + date + ")";
+			name += " (merged)";
+
+			// For good measure, get all sources
+			Dictionary<int, string> sources = new Dictionary<int, string>();
+			sources.Add(0, "Default");
+
+			string squery = "SELECT id, name FROM source";
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+
+				using (SqliteCommand slc = new SqliteCommand(squery, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						while (sldr.Read())
+						{
+							sources.Add(sldr.GetInt32(0), sldr.GetString(1));
+						}
+					}
+				}
+			}
+
+			// Get a list of files to sourceid mappings
+			Dictionary<string, string> sourcemap = new Dictionary<string, string>();
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+
+				string tquery = "SELECT DISTINCT dats.sha1, datsdata.value FROM dats JOIN datsdata ON dats.id=datsdata.id WHERE key='source'";
+				using (SqliteCommand slc = new SqliteCommand(tquery, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						while (sldr.Read())
+						{
+							string tempsha1 = sldr.GetString(0);
+							string tempval = sldr.GetString(1);
+							if (!sourcemap.ContainsKey(tempsha1))
+							{
+								sourcemap.Add(tempsha1, tempval);
+							}
+						}
+					}
+				}
+			}
+
+			// Create the output DatData object
+			Dat datdata = new Dat
+			{
+				FileName = description,
+				Name = name,
+				Description = description,
+				Version = "",
+				Date = date,
+				Category = "SabreTools",
+				Author = "SabreTools",
+				ForcePacking = ForcePacking.None,
+				OutputFormat = (_old ? OutputFormat.ClrMamePro : OutputFormat.Xml),
+				MergeRoms = true,
+			};
+
+			// Now read in all of the files
+			SHA1 sha1 = SHA1.Create();
+			foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories))
+			{
+				string hash = "";
+				using (FileStream fs = File.Open(file, FileMode.Open))
+				{
+					hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", "");
+				}
+
+				int tempSrcId = 0;
+				if (sourcemap.ContainsKey(hash))
+				{
+					Int32.TryParse(sourcemap[hash], out tempSrcId);
+				}
+				datdata = DatTools.Parse(file, 0, tempSrcId, datdata, _logger);
+			}
+
+			// If the dictionary is empty for any reason, tell the user and exit
+			if (datdata.Files.Keys.Count == 0 || datdata.Files.Count == 0)
+			{
+				_logger.Log("No roms found for system ID " + _systemid);
+				return false;
+			}
+
+			// Now process all of the roms
+			_logger.User("Cleaning rom data");
+			List<string> keys = datdata.Files.Keys.ToList();
+			foreach (string key in keys)
+			{
+				List<Rom> temp = new List<Rom>();
+				List<Rom> newroms = datdata.Files[key];
+				for (int i = 0; i < newroms.Count; i++)
+				{
+					Rom rom = newroms[i];
+
+					// In the case that the RomData is incomplete, skip it
+					if (rom.Name == null || rom.Machine.Name == null)
+					{
+						continue;
+					}
+
+					// WOD origninally stripped out any subdirs from the imported files, we do the same
+					rom.Name = Path.GetFileName(rom.Name);
+
+					// Run the name through the filters to make sure that it's correct
+					rom.Name = Style.NormalizeChars(rom.Name);
+					rom.Name = Style.RussianToLatin(rom.Name);
+					rom.Name = Regex.Replace(rom.Name, @"(.*) \.(.*)", "$1.$2");
+
+					// Run the name through the filters to make sure that it's correct
+					rom.Machine.Name = Style.NormalizeChars(rom.Machine.Name);
+					rom.Machine.Name = Style.RussianToLatin(rom.Machine.Name);
+					rom.Machine.Name = Style.SearchPattern(rom.Machine.Name);
+
+					// WoD gets rid of anything past the first "(" or "[" as the name, we will do the same
+					string stripPattern = @"(([[(].*[\)\]] )?([^([]+))";
+					Regex stripRegex = new Regex(stripPattern);
+					Match stripMatch = stripRegex.Match(rom.Machine.Name);
+					rom.Machine.Name = stripMatch.Groups[1].Value;
+
+					rom.Machine.Name = rom.Machine.Name.TrimEnd().TrimStart();
+
+					if (!_norename)
+					{
+						rom.Machine.Name += " [" + sources[rom.Metadata.SourceID] + "]";
+					}
+
+					// If a game has source "0" it's Default. Make this Int32.MaxValue for sorting purposes
+					if (rom.Metadata.SourceID == 0)
+					{
+						rom.Metadata.SourceID = Int32.MaxValue;
+					}
+
+					temp.Add(rom);
+				}
+				datdata.Files[key] = temp;
+			}
+
+			// Then write out the file
+			DatTools.WriteDatfile(datdata, _outroot, _logger, _norename);
+
+			return true;
+		}
+	}
+}

SabreTools.Helper/Objects/Import.cs

@@ -0,0 +1,652 @@
+using Mono.Data.Sqlite;
+using SabreTools.Helper;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.RegularExpressions;
+
+namespace SabreTools
+{
+	/// <summary>
+	/// Import data into the database from existing DATs
+	/// </summary>
+	public class Import : IImport
+	{
+		// Private instance variables
+		private string _filepath;
+		private string _connectionString;
+		private Logger _logger;
+
+		/// <summary>
+		/// Initialize an Import object with the given information
+		/// </summary>
+		/// <param name="filepath">Path to the file that is going to be imported</param>
+		/// <param name="connectionString">Connection string for SQLite</param>
+		/// <param name="logger">Logger object for file or console output</param>
+		public Import(string filepath, string connectionString, Logger logger)
+		{
+			_filepath = filepath;
+			_connectionString = connectionString;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Import the data from file into the database
+		/// </summary>
+		/// <returns>True if the data was imported, false otherwise</returns>
+		public bool UpdateDatabase()
+		{
+			// If file doesn't exist, error and return
+			if (!File.Exists(_filepath))
+			{
+				_logger.Error("File '" + _filepath + "' doesn't exist");
+				return false;
+			}
+
+			// Determine which dattype we have
+			string filename = Path.GetFileName(_filepath);
+			GroupCollection fileinfo;
+			DatType type = DatType.none;
+
+			if (Regex.IsMatch(filename, Constants.NonGoodPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NonGoodPattern).Groups;
+				type = DatType.NonGood;
+			}
+			else if (Regex.IsMatch(filename, Constants.NonGoodSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NonGoodSpecialPattern).Groups;
+				type = DatType.NonGood;
+			}
+			else if (Regex.IsMatch(filename, Constants.GoodPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.GoodPattern).Groups;
+				type = DatType.Good;
+			}
+			else if (Regex.IsMatch(filename, Constants.GoodXmlPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.GoodXmlPattern).Groups;
+				type = DatType.Good;
+			}
+			else if (Regex.IsMatch(filename, Constants.MaybeIntroPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.MaybeIntroPattern).Groups;
+				type = DatType.MaybeIntro;
+			}
+			else if (Regex.IsMatch(filename, Constants.NoIntroPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroPattern).Groups;
+				type = DatType.NoIntro;
+			}
+			// For numbered DATs only
+			else if (Regex.IsMatch(filename, Constants.NoIntroNumberedPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroNumberedPattern).Groups;
+				type = DatType.NoIntro;
+			}
+			// For N-Gage and Gizmondo only
+			else if (Regex.IsMatch(filename, Constants.NoIntroSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroSpecialPattern).Groups;
+				type = DatType.NoIntro;
+			}
+			else if (Regex.IsMatch(filename, Constants.RedumpPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.RedumpPattern).Groups;
+				type = DatType.Redump;
+			}
+			// For special BIOSes only
+			else if (Regex.IsMatch(filename, Constants.RedumpBiosPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.RedumpBiosPattern).Groups;
+				type = DatType.Redump;
+			}
+			else if (Regex.IsMatch(filename, Constants.TosecPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.TosecPattern).Groups;
+				type = DatType.TOSEC;
+			}
+			else if (Regex.IsMatch(filename, Constants.TruripPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.TruripPattern).Groups;
+				type = DatType.TruRip;
+			}
+			else if (Regex.IsMatch(filename, Constants.ZandroPattern))
+			{
+				filename = "Nintendo - Super Nintendo Entertainment System (Zandro " + File.GetLastWriteTime(_filepath).ToString("yyyyMMddHHmmss") + ").dat";
+				fileinfo = Regex.Match(filename, Constants.DefaultPattern).Groups;
+				type = DatType.Custom;
+			}
+			else if (Regex.IsMatch(filename, Constants.DefaultPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.DefaultPattern).Groups;
+				type = DatType.Custom;
+			}
+			else if (Regex.IsMatch(filename, Constants.DefaultSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.DefaultSpecialPattern).Groups;
+				type = DatType.Custom;
+			}
+			else if (Regex.IsMatch(filename, Constants.MamePattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.MamePattern).Groups;
+				type = DatType.MAME;
+			}
+			// If the type is still unmatched, the data can't be imported yet
+			else
+			{
+				_logger.Warning("File " + filename + " cannot be imported at this time because it is not a known pattern.\nPlease try again with an unrenamed version.");
+				return false;
+			}
+
+			_logger.Log("Type detected: " + type.ToString());
+
+			// Check for and extract import information from the file name based on type
+			string manufacturer = "";
+			string system = "";
+			string source = "";
+			string datestring = "";
+			string date = "";
+
+			switch (type)
+			{
+				case DatType.Good:
+					if (!Mappings.DatMaps["Good"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection goodInfo = Regex.Match(Mappings.DatMaps["Good"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = goodInfo[1].Value;
+					system = goodInfo[2].Value;
+					source = "Good";
+					date = File.GetLastWriteTime(_filepath).ToString("yyyy-MM-dd HH:mm:ss");
+					break;
+				case DatType.MAME:
+					if (!Mappings.DatMaps["MAME"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection mameInfo = Regex.Match(Mappings.DatMaps["MAME"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = mameInfo[1].Value;
+					system = mameInfo[2].Value;
+					source = "MAME";
+					date = File.GetLastWriteTime(_filepath).ToString("yyyy-MM-dd HH:mm:ss");
+					break;
+				case DatType.MaybeIntro:
+					if (!Mappings.DatMaps["MaybeIntro"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection maybeIntroInfo = Regex.Match(Mappings.DatMaps["MaybeIntro"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = maybeIntroInfo[1].Value;
+					system = maybeIntroInfo[2].Value;
+					source = "Maybe-Intro";
+					datestring = fileinfo[2].Value;
+					GroupCollection miDateInfo = Regex.Match(datestring, Constants.NoIntroSpecialDatePattern).Groups;
+					date = miDateInfo[1].Value + "-" + miDateInfo[2].Value + "-" + miDateInfo[3].Value + " 00:00:00";
+					break;
+				case DatType.NoIntro:
+					if (!Mappings.DatMaps["NoIntro"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection nointroInfo = Regex.Match(Mappings.DatMaps["NoIntro"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = nointroInfo[1].Value;
+					system = nointroInfo[2].Value;
+					source = "no-Intro";
+					if (fileinfo.Count < 2)
+					{
+						date = File.GetLastWriteTime(_filepath).ToString("yyyy-MM-dd HH:mm:ss");
+					}
+					else if (Regex.IsMatch(fileinfo[2].Value, Constants.NoIntroDatePattern))
+					{
+						datestring = fileinfo[2].Value;
+						GroupCollection niDateInfo = Regex.Match(datestring, Constants.NoIntroDatePattern).Groups;
+						date = niDateInfo[1].Value + "-" + niDateInfo[2].Value + "-" + niDateInfo[3].Value + " " +
+							niDateInfo[4].Value + ":" + niDateInfo[5].Value + ":" + niDateInfo[6].Value;
+					}
+					else
+					{
+						datestring = fileinfo[2].Value;
+						GroupCollection niDateInfo = Regex.Match(datestring, Constants.NoIntroSpecialDatePattern).Groups;
+						date = niDateInfo[1].Value + "-" + niDateInfo[2].Value + "-" + niDateInfo[3].Value + " 00:00:00";
+					}
+					break;
+				case DatType.NonGood:
+					if (!Mappings.DatMaps["NonGood"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection nonGoodInfo = Regex.Match(Mappings.DatMaps["NonGood"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = nonGoodInfo[1].Value;
+					system = nonGoodInfo[2].Value;
+					source = "NonGood";
+					date = File.GetLastWriteTime(_filepath).ToString("yyyy-MM-dd HH:mm:ss");
+					break;
+				case DatType.Redump:
+					if (!Mappings.DatMaps["Redump"].ContainsKey(fileinfo[1].Value))
+					{
+						// Handle special case mappings found only in Redump
+						fileinfo = Regex.Match(filename, Constants.RedumpBiosPattern).Groups;
+
+						if (!Mappings.DatMaps["Redump"].ContainsKey(fileinfo[1].Value))
+						{
+							_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+							return false;
+						}
+					}
+					GroupCollection redumpInfo = Regex.Match(Mappings.DatMaps["Redump"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = redumpInfo[1].Value;
+					system = redumpInfo[2].Value;
+					source = "Redump";
+					datestring = fileinfo[2].Value;
+					if (Regex.IsMatch(datestring, Constants.RedumpDatePattern))
+					{
+						GroupCollection rdDateInfo = Regex.Match(datestring, Constants.RedumpDatePattern).Groups;
+						date = rdDateInfo[1].Value + "-" + rdDateInfo[2].Value + "-" + rdDateInfo[3].Value + " " +
+							rdDateInfo[4].Value + ":" + rdDateInfo[5].Value + ":" + rdDateInfo[6].Value;
+					}
+					else
+					{
+						GroupCollection rdDateInfo = Regex.Match(datestring, Constants.TosecDatePattern).Groups;
+						date = rdDateInfo[1].Value + "-" + rdDateInfo[2].Value + "-" + rdDateInfo[3].Value + " 00:00:00";
+					}
+					
+					break;
+				case DatType.TOSEC:
+					if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+					{
+						// Handle special case mappings found only in TOSEC
+						fileinfo = Regex.Match(filename, Constants.TosecSpecialPatternA).Groups;
+
+						if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+						{
+							fileinfo = Regex.Match(filename, Constants.TosecSpecialPatternB).Groups;
+
+							if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+							{
+								_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+								return false;
+							}
+						}
+					}
+					GroupCollection tosecInfo = Regex.Match(Mappings.DatMaps["TOSEC"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = tosecInfo[1].Value;
+					system = tosecInfo[2].Value;
+					source = "TOSEC";
+					datestring = fileinfo[2].Value;
+					GroupCollection toDateInfo = Regex.Match(datestring, Constants.TosecDatePattern).Groups;
+					date = toDateInfo[1].Value + "-" + toDateInfo[2].Value + "-" + toDateInfo[3].Value + " 00:00:00";
+					break;
+				case DatType.TruRip:
+					if (!Mappings.DatMaps["TruRip"].ContainsKey(fileinfo[1].Value))
+					{
+						_logger.Warning("The filename " + fileinfo[1].Value + " could not be mapped! Please check the mappings and try again");
+						return false;
+					}
+					GroupCollection truripInfo = Regex.Match(Mappings.DatMaps["TruRip"][fileinfo[1].Value], Constants.RemappedPattern).Groups;
+
+					manufacturer = truripInfo[1].Value;
+					system = truripInfo[2].Value;
+					source = "trurip";
+					date = File.GetLastWriteTime(_filepath).ToString("yyyy-MM-dd HH:mm:ss");
+					break;
+				case DatType.Custom:
+				default:
+					manufacturer = fileinfo[1].Value;
+					system = fileinfo[2].Value;
+					source = fileinfo[3].Value;
+					datestring = fileinfo[4].Value;
+
+					GroupCollection cDateInfo = Regex.Match(datestring, Constants.DefaultDatePattern).Groups;
+					date = cDateInfo[1].Value + "-" + cDateInfo[2].Value + "-" + cDateInfo[3].Value + " " +
+						cDateInfo[4].Value + ":" + cDateInfo[5].Value + ":" + cDateInfo[6].Value;
+					break;
+			}
+
+			// Check to make sure that the manufacturer and system are valid according to the database
+			int sysid = -1;
+			string query = "SELECT id FROM systems WHERE manufacturer='" + manufacturer + "' AND system='" + system +"'";
+			//string query = "SELECT id FROM system WHERE manufacturer='" + manufacturer + "' AND name='" + system + "'";
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						// If nothing is found, tell the user and exit
+						if (!sldr.HasRows)
+						{
+							_logger.Error("No suitable system for '" + filename + "' (" + manufacturer + " " + system + ") found! Please add the system and then try again.");
+							return false;
+						}
+
+						// Set the system ID from the first found value
+						sldr.Read();
+						sysid = sldr.GetInt32(0);
+					}
+				}
+			}
+
+			// Check to make sure that the source is valid according to the database
+			int srcid = -1;
+			query = "SELECT id FROM sources WHERE name='" + source + "'";
+			//query = "SELECT id FROM source WHERE name='" + source + "'";
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						// If nothing is found, tell the user and exit
+						if (!sldr.HasRows)
+						{
+							_logger.Error("No suitable source for '" + filename + "' found! Please add the source and then try again.");
+							return false;
+						}
+
+						// Set the source ID from the first found value
+						sldr.Read();
+						srcid = sldr.GetInt32(0);
+					}
+				}
+			}
+
+			// Get all roms that are found in the DAT to see what needs to be added
+			Dat datdata = new Dat();
+			datdata = DatTools.Parse(_filepath, sysid, srcid, datdata, _logger);
+
+			// Sort inputted roms into games
+			SortedDictionary<string, List<Rom>> sortable = new SortedDictionary<string, List<Rom>>();
+			long count = 0;
+			foreach (List<Rom> roms in datdata.Files.Values)
+			{
+				List<Rom> newroms = roms;
+				if (datdata.MergeRoms)
+				{
+					newroms = RomTools.Merge(newroms, _logger);
+				}
+
+				foreach (Rom rom in newroms)
+				{
+					count++;
+					string key = rom.Metadata.SystemID.ToString().PadLeft(10, '0') + "-" + rom.Metadata.SourceID.ToString().PadLeft(10, '0') + "-" + rom.Machine.Name.ToLowerInvariant();
+					if (sortable.ContainsKey(key))
+					{
+						sortable[key].Add(rom);
+					}
+					else
+					{
+						List<Rom> temp = new List<Rom>();
+						temp.Add(rom);
+						sortable.Add(key, temp);
+					}
+				}
+			}
+
+			// Loop over all roms, checking for adds
+			foreach (string key in sortable.Keys)
+			{
+				List<Rom> roms = sortable[key];
+				RomTools.Sort(roms, true);
+
+				long gameid = -1;
+				using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+				{
+					dbc.Open();
+
+					// For each game, check for a new ID
+					gameid = AddGame(sysid, roms[0].Machine.Name, srcid, dbc);
+
+					foreach (Rom rom in roms)
+					{
+						// BEGIN COMMENT
+						// Try to add the rom with the game information
+						AddRom(rom, gameid, date, dbc);
+						// END COMMENT
+
+						/*
+						// Try to add the romdata
+						AddHash(rom, sysid, srcid, date, dbc);
+						*/
+					}
+				}
+			}
+
+			return true;
+		}
+
+		/// <summary>
+		/// Add a game to the database if it doesn't already exist
+		/// </summary>
+		/// <param name="sysid">System ID for the game to be added with</param>
+		/// <param name="machinename">Name of the game to be added</param>
+		/// <param name="srcid">Source ID for the game to be added with</param>
+		/// <param name="dbc">SQLite database connection to use</param>
+		/// <returns>Game ID of the inserted (or found) game, -1 on error</returns>
+		private long AddGame(int sysid, string machinename, int srcid, SqliteConnection dbc)
+		{
+			// WoD gets rid of anything past the first "(" or "[" as the name, we will do the same
+			string stripPattern = @"(([[(].*[\)\]] )?([^([]+))";
+			Regex stripRegex = new Regex(stripPattern);
+			Match stripMatch = stripRegex.Match(machinename);
+			machinename = stripMatch.Groups[1].Value;
+
+			// Run the name through the filters to make sure that it's correct
+			machinename = Style.NormalizeChars(machinename);
+			machinename = Style.RussianToLatin(machinename);
+			machinename = Style.SearchPattern(machinename);
+			machinename = machinename.Trim();
+
+			long gameid = -1;
+			string query = "SELECT id FROM games WHERE system=" + sysid +
+				" AND name='" + machinename.Replace("'", "''") + "'" +
+				" AND source=" + srcid;
+
+			using (SqliteCommand slc = new SqliteCommand(query, dbc))
+			{
+				using (SqliteDataReader sldr = slc.ExecuteReader())
+				{
+					// If nothing is found, add the game and get the insert ID
+					if (!sldr.HasRows)
+					{
+						query = "INSERT INTO games (system, name, source)" +
+							" VALUES (" + sysid + ", '" + machinename.Replace("'", "''") + "', " + srcid + ")";
+
+						using (SqliteCommand slc2 = new SqliteCommand(query, dbc))
+						{
+							slc2.ExecuteNonQuery();
+						}
+
+						query = "SELECT last_insertConstants.Rowid()";
+						using (SqliteCommand slc2 = new SqliteCommand(query, dbc))
+						{
+							gameid = (long)slc2.ExecuteScalar();
+						}
+					}
+					// Otherwise, retrieve the ID
+					else
+					{
+						sldr.Read();
+						gameid = sldr.GetInt64(0);
+					}
+				}
+			}
+
+			return gameid;
+		}
+
+		/// <summary>
+		/// Add a file to the database if it doesn't already exist
+		/// </summary>
+		/// <param name="rom">Rom object representing the rom</param>
+		/// <param name="gameid">ID of the parent game to be mapped to</param>
+		/// <param name="date">Last updated date</param>
+		/// <param name="dbc">SQLite database connection to use</param>
+		/// <returns>True if the file exists or could be added, false on error</returns>
+		private bool AddRom(Rom rom, long gameid, string date, SqliteConnection dbc)
+		{
+			// WOD origninally stripped out any subdirs from the imported files, we do the same
+			rom.Name = Path.GetFileName(rom.Name);
+
+			// Run the name through the filters to make sure that it's correct
+			rom.Name = Style.NormalizeChars(rom.Name);
+			rom.Name = Style.RussianToLatin(rom.Name);
+			rom.Name = Regex.Replace(rom.Name, @"(.*) \.(.*)", "$1.$2");
+
+			if (rom.Type != ItemType.Rom && rom.Type != ItemType.Disk)
+			{
+				rom.Type = ItemType.Rom;
+			}
+
+			// Check to see if this exact file is in the database already
+			string query = @"
+SELECT files.id FROM files
+	JOIN checksums
+	ON files.id=checksums.file
+	WHERE files.name='" + rom.Name.Replace("'", "''") + @"'
+		AND files.type='" + rom.Type + @"' 
+		AND files.setid=" + gameid +
+		" AND checksums.size=" + rom.HashData.Size +
+		" AND checksums.crc='" + rom.HashData.CRC + "'" +
+		" AND checksums.md5='" + rom.HashData.MD5 + "'" +
+		" AND checksums.sha1='" + rom.HashData.SHA1 + "'";
+
+			using (SqliteCommand slc = new SqliteCommand(query, dbc))
+			{
+				using (SqliteDataReader sldr = slc.ExecuteReader())
+				{
+					// If the file doesn't exist, add it with its checksums
+					if (!sldr.HasRows)
+					{
+						query = @"BEGIN;
+INSERT INTO files (setid, name, type, lastupdated)
+VALUES (" + gameid + ", '" + rom.Name.Replace("'", "''") + "', '" + rom.Type + "', '" + date + @"');
+INSERT INTO checksums (file, size, crc, md5, sha1)
+VALUES ((SELECT last_insertConstants.Rowid()), " + rom.HashData.Size + ", '" + rom.HashData.CRC + "'" + ", '" + rom.HashData.MD5 + "'" + ", '" + rom.HashData.SHA1 + @"');
+COMMIT;";
+						using (SqliteCommand slc2 = new SqliteCommand(query, dbc))
+						{
+							int affected = slc2.ExecuteNonQuery();
+
+							// If the insert was unsuccessful, something bad happened
+							if (affected < 1)
+							{
+								_logger.Error("There was an error adding " + rom.Name + " to the database!");
+								return false;
+							}
+						}
+					}
+				}
+			}
+
+			return true;
+		}
+
+		/// <summary>
+		/// Add a hash to the database if it doesn't exist already
+		/// </summary>
+		/// <param name="rom">Rom object representing the rom</param>
+		/// <param name="sysid">System ID for the game to be added with</param>
+		/// <param name="srcid">Source ID for the game to be added with</param>
+		/// <param name="date">Last updated date</param>
+		/// <param name="dbc">SQLite database connection to use</param>
+		/// <returns>True if the hash exists or could be added, false on error</returns>
+		/// <remarks>This is currently unused. It is a test method for the new SabreTools DB schema</remarks>
+		private bool AddHash(Rom rom, int sysid, int srcid, string date, SqliteConnection dbc)
+		{
+			// Process the game name
+
+			// WoD gets rid of anything past the first "(" or "[" as the name, we will do the same
+			string stripPattern = @"(([[(].*[\)\]] )?([^([]+))";
+			Regex stripRegex = new Regex(stripPattern);
+			Match stripMatch = stripRegex.Match(rom.Machine.Name);
+			rom.Machine.Name = stripMatch.Groups[1].Value;
+
+			// Run the name through the filters to make sure that it's correct
+			rom.Machine.Name = Style.NormalizeChars(rom.Machine.Name);
+			rom.Machine.Name = Style.RussianToLatin(rom.Machine.Name);
+			rom.Machine.Name = Style.SearchPattern(rom.Machine.Name);
+			rom.Machine.Name = rom.Machine.Name.Trim();
+
+			// Process the rom name
+
+			// WOD origninally stripped out any subdirs from the imported files, we do the same
+			rom.Name = Path.GetFileName(rom.Name);
+
+			// Run the name through the filters to make sure that it's correct
+			rom.Name = Style.NormalizeChars(rom.Name);
+			rom.Name = Style.RussianToLatin(rom.Name);
+			rom.Name = Regex.Replace(rom.Name, @"(.*) \.(.*)", "$1.$2");
+
+			// Retrieve or insert the hash
+			long hashid = -1;
+			string query = "SELECT id FROM hash WHERE size=" + rom.HashData.Size + " AND crc='" + rom.HashData.CRC + "' AND md5='" + rom.HashData.MD5 + "' AND sha1='" + rom.HashData.SHA1 + "'";
+			using (SqliteCommand slc = new SqliteCommand(query, dbc))
+			{
+				using (SqliteDataReader sldr = slc.ExecuteReader())
+				{
+					// If nothing is found, add the hash and get the insert ID
+					if (!sldr.HasRows)
+					{
+						query = "INSERT INTO hash (size, crc, md5, sha1)" +
+							" VALUES (" + rom.HashData.Size + ", '" + rom.HashData.CRC + "', '" + rom.HashData.MD5 + "', '" + rom.HashData.SHA1 + "')";
+
+						using (SqliteCommand slc2 = new SqliteCommand(query, dbc))
+						{
+							slc2.ExecuteNonQuery();
+						}
+
+						query = "SELECT last_insertConstants.Rowid()";
+						using (SqliteCommand slc2 = new SqliteCommand(query, dbc))
+						{
+							hashid = (long)slc2.ExecuteScalar();
+						}
+					}
+					// Otherwise, retrieve the ID
+					else
+					{
+						sldr.Read();
+						hashid = sldr.GetInt64(0);
+					}
+				}
+			}
+
+			// Ignore or insert the file and game
+			query = @"BEGIN;
+INSERT OR IGNORE INTO hashdata (hashid, key, value) VALUES " +
+	"(" + hashid + ", 'name', '" + rom.Name.Replace("'", "''") + "'), " +
+	"(" + hashid + ", 'game', '" + rom.Machine.Name.Replace("'", "''") + "'), " +
+	"(" + hashid + ", 'type', '" + rom.Type + "'), " +
+	"(" + hashid + ", 'lastupdated', '" + date + @"');
+INSERT OR IGNORE INTO gamesystem (game, systemid) VALUES ('" + rom.Machine.Name.Replace("'", "''") + "', " + sysid + @");
+INSERT OR IGNORE INTO gamesource (game, sourceid) VALUES ('" + rom.Machine.Name.Replace("'", "''") + "', " + srcid + @");
+COMMIT;";
+
+			using (SqliteCommand slc = new SqliteCommand(query, dbc))
+			{
+				int ret = slc.ExecuteNonQuery();
+				if ((SQLiteErrorCode)ret == SQLiteErrorCode.Error)
+				{
+					_logger.Error("A SQLite error has occurred: " + ((SQLiteErrorCode)ret).ToString());
+					return false;
+				}
+			}
+
+			return true;
+		}
+	}
+}

SabreTools.Helper/Objects/ImportTwo.cs

@@ -0,0 +1,455 @@
+using Mono.Data.Sqlite;
+using SabreTools.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text.RegularExpressions;
+
+namespace SabreTools
+{
+	public class ImportTwo : IImport
+	{
+		// Private instance variables
+		private string _datroot;
+		private string _connectionString;
+		private Logger _logger;
+		private bool _ignore;
+
+		/// <summary>
+		/// Initialize an Import object with the given information
+		/// </summary>
+		/// <param name="datroot">Root directory where all DAT files are held</param>
+		/// <param name="connectionString">Connection string for SQLite</param>
+		/// <param name="logger">Logger object for file or console output</param>
+		/// <param name="ignore">False if each DAT that has no defined source asks for user input (default), true otherwise</param>
+		public ImportTwo(string datroot, string connectionString, Logger logger, bool ignore = false)
+		{
+			_datroot = datroot;
+			_connectionString = connectionString;
+			_logger = logger;
+			_ignore = ignore;
+		}
+
+		/// <summary>
+		/// Perform initial or incremental import of DATs in the root folder
+		/// </summary>
+		/// <returns>True if the data could be inserted or updated correctly, false otherwise</returns>
+		public bool UpdateDatabase()
+		{
+			_logger.User("Beginning import/update process");
+
+			Dictionary<Tuple<long, string, string>, int> missing = ImportData();
+			bool success = RemoveData(missing);
+
+			_logger.User("Import/update process complete!");
+
+			return success;
+		}
+
+		/// <summary>
+		/// Import data into the database and return all files not found in the list
+		/// </summary>
+		/// <returns>List of files that were not found in the audit</returns>
+		private Dictionary<Tuple<long, string, string>, int> ImportData()
+		{
+			// Create the empty dictionary for file filtering and output
+			Dictionary<Tuple<long, string, string>, int> dbfiles = new Dictionary<Tuple<long, string, string>, int>();
+
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+				_logger.User("Populating reference objects");
+
+				// Populate the list of files in the database with Tuples (size, sha1, name)
+				string query = "SELECT id, size, sha1, name FROM dats";
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						while (sldr.Read())
+						{
+							dbfiles.Add(Tuple.Create(sldr.GetInt64(1), sldr.GetString(2), sldr.GetString(3)), sldr.GetInt32(0));
+						}
+					}
+				}
+
+				// Populate the list of systems
+				Dictionary<string, int> systems = new Dictionary<string, int>();
+				query = "SELECT id, manufacturer, name FROM system";
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader sldr = slc.ExecuteReader())
+					{
+						while (sldr.Read())
+						{
+							systems.Add(sldr.GetString(1) + " - " + sldr.GetString(2), sldr.GetInt32(0));
+						}
+					}
+				}
+
+				// Populate the list of sources (initial)
+				SortedDictionary<string, int> sources = new SortedDictionary<string, int>();
+				sources.Add("default", 0);
+				query = "SELECT name, id FROM source";
+				using (SqliteCommand sslc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader ssldr = sslc.ExecuteReader())
+					{
+						while (ssldr.Read())
+						{
+							sources.Add(ssldr.GetString(0).ToLowerInvariant(), ssldr.GetInt32(1));
+						}
+					}
+				}
+
+				// Interate through each system and check files
+				SHA1 sha1 = SHA1.Create();
+				foreach (KeyValuePair<string, int> kv in systems)
+				{
+					_logger.User("Processing DATs for system: '" + kv.Key + "'");
+
+					// Set the folder to iterate through based on the DAT root
+					string folder = Path.Combine(_datroot, kv.Key.Trim());
+
+					// Audit all files in the folder
+					foreach (string file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories))
+					{
+						// First get the file information for comparison
+						long size = (new FileInfo(file)).Length;
+						string hash = "";
+						using (FileStream fs = File.Open(file, FileMode.Open))
+						{
+							hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", "");
+						}
+
+						// If it's not in the list of known files, add it
+						if (!dbfiles.ContainsKey(Tuple.Create(size, hash, file)))
+						{
+							// First add the information to the database as is and return the new insert ID
+							_logger.Log("Adding file information for " + Path.GetFileName(file));
+
+							int hashid = -1;
+							query = @"INSERT INTO dats (size, sha1, name)
+VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"')";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
+							{
+								slc.ExecuteNonQuery();
+							}
+
+							query = "SELECT last_insert_rowid()";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
+							{
+								using (SqliteDataReader sldr = slc.ExecuteReader())
+								{
+									if (sldr.Read())
+									{
+										hashid = sldr.GetInt32(0);
+									}
+								}
+							}
+
+							// Next we try to figure out the source ID from the name
+							string possiblesource = GetSourceFromFileName(Path.GetFileName(file));
+
+							// Try to get the source ID from the name
+							int sourceid = (sources.ContainsKey(possiblesource.ToLowerInvariant()) ? sources[possiblesource] : 0);
+
+							// If we have a "default" ID and we're not ignoring new sources, prompt for a source input
+							if (!_ignore && sourceid <= 1)
+							{
+								// We want to reset "Default" at this point, just in case
+								if (possiblesource.ToLowerInvariant() == "default")
+								{
+									possiblesource = "";
+								}
+
+								// If the source is blank, ask the user to supply one
+								while (possiblesource == "" && sourceid == 0)
+								{
+									Console.Clear();
+									Build.Start("DATabaseTwo");
+
+									Console.WriteLine("Sources:");
+									foreach (KeyValuePair<string, int> pair in sources)
+									{
+										Console.WriteLine("    " + pair.Value + " - " + Style.SentenceCase(pair.Key));
+									}
+									Console.WriteLine("\nFor file name: " + Path.GetFileName(file));
+									Console.Write("Select a source above or enter a new one: ");
+									possiblesource = Console.ReadLine();
+
+									Int32.TryParse(possiblesource, out sourceid);
+
+									// If the value could be parsed, reset the source string
+									if (sourceid != 0)
+									{
+										possiblesource = "";
+									}
+
+									// If the source ID is set check to see if it's valid
+									if (sourceid != 0 && !sources.ContainsValue(sourceid))
+									{
+										Console.WriteLine("Invalid selection: " + sourceid);
+										Console.ReadLine();
+										sourceid = 0;
+									}
+								}
+
+								// If we have a non-empty possible source and it's in the database, get the id
+								if (possiblesource != "" && sources.ContainsKey(possiblesource.ToLowerInvariant()))
+								{
+									sourceid = sources[possiblesource.ToLowerInvariant()];
+								}
+
+								// If we have a non-empty possible source and it's not in the database, insert and get the id
+								else if (possiblesource != "" && !sources.ContainsKey(possiblesource.ToLowerInvariant()))
+								{
+									query = @"BEGIN;
+INSERT INTO source (name, url)
+VALUES ('" + possiblesource + @"', '');
+SELECT last_insertConstants.Rowid();
+COMMIT;";
+									using (SqliteCommand slc = new SqliteCommand(query, dbc))
+									{
+										using (SqliteDataReader sldr = slc.ExecuteReader())
+										{
+											if (sldr.Read())
+											{
+												sourceid = sldr.GetInt32(0);
+											}
+										}
+									}
+
+									// Add the new source to the current dictionary
+									sources.Add(possiblesource.ToLowerInvariant(), sourceid);
+								}
+							}
+
+							// Now that we have a source ID, we can add the mappings for system and source to the database
+							query = @"INSERT OR IGNORE INTO datsdata (id, key, value)
+VALUES (" + hashid + ", 'source', '" + sourceid + @"'),
+(" + hashid + ", 'system', '" + kv.Value + "')";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
+							{
+								slc.ExecuteNonQuery();
+							}
+						}
+
+						// Otherwise, remove it from the list of found items
+						else
+						{
+							dbfiles.Remove(Tuple.Create(size, hash, file));
+						}
+					}
+				}
+			}
+
+			return dbfiles;
+		}
+
+		/// <summary>
+		/// Remove all data associated with various files
+		/// </summary>
+		/// <param name="missing">List of file identifiers to remove from the database</param>
+		/// <returns>True if everything went well, false otherwise</returns>
+		private bool RemoveData(Dictionary<Tuple<long, string, string>, int> missing)
+		{
+			bool success = true;
+
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+
+				// Get a comma-separated list of IDs from the input files
+				string idlist = String.Join(",", missing.Values);
+
+				// Now remove all of the files from the database
+				string query = @"BEGIN;
+DELETE FROM datsdata WHERE id IN (" + idlist + @");
+DELETE FROM dats WHERE id IN (" + idlist + @");
+COMMIT;";
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					slc.ExecuteNonQuery();
+				}
+			}
+
+			return success;
+		}
+
+		/// <summary>
+		/// Determine the source name from the file name, if possible
+		/// </summary>
+		/// <param name="filename">The name of the file to be checked</param>
+		/// <returns>The name of the source if determined, blank otherwise</returns>
+		private string GetSourceFromFileName(string filename)
+		{
+			string source = "Default";
+
+			// Determine which dattype we have
+			GroupCollection fileinfo;
+
+			if (Regex.IsMatch(filename, Constants.NonGoodPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NonGoodPattern).Groups;
+				if (!Mappings.DatMaps["NonGood"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped.");
+					return source;
+				}
+				source = "NonGood";
+			}
+			else if (Regex.IsMatch(filename, Constants.NonGoodSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NonGoodSpecialPattern).Groups;
+				if (!Mappings.DatMaps["NonGood"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped.");
+					return source;
+				}
+				source = "NonGood";
+			}
+			else if (Regex.IsMatch(filename, Constants.GoodPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.GoodPattern).Groups;
+				if (!Mappings.DatMaps["Good"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped.");
+					return source;
+				}
+				source = "Good";
+			}
+			else if (Regex.IsMatch(filename, Constants.GoodXmlPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.GoodXmlPattern).Groups;
+				if (!Mappings.DatMaps["Good"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped.");
+					return source;
+				}
+				source = "Good";
+			}
+			else if (Regex.IsMatch(filename, Constants.MaybeIntroPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.MaybeIntroPattern).Groups;
+				if (!Mappings.DatMaps["MaybeIntro"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Maybe-Intro but could not be mapped.");
+					return source;
+				}
+				source = "Maybe-Intro";
+			}
+			else if (Regex.IsMatch(filename, Constants.NoIntroPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroPattern).Groups;
+				if (!Mappings.DatMaps["NoIntro"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped.");
+					return source;
+				}
+				source = "no-Intro";
+			}
+			// For numbered DATs only
+			else if (Regex.IsMatch(filename, Constants.NoIntroNumberedPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroNumberedPattern).Groups;
+				if (!Mappings.DatMaps["NoIntro"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped.");
+					return source;
+				}
+				source = "no-Intro";
+			}
+			// For N-Gage and Gizmondo only
+			else if (Regex.IsMatch(filename, Constants.NoIntroSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.NoIntroSpecialPattern).Groups;
+				if (!Mappings.DatMaps["NoIntro"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped.");
+					return source;
+				}
+				source = "no-Intro";
+			}
+			else if (Regex.IsMatch(filename, Constants.RedumpPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.RedumpPattern).Groups;
+				if (!Mappings.DatMaps["Redump"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped.");
+					return source;
+				}
+				source = "Redump";
+			}
+			// For special BIOSes only
+			else if (Regex.IsMatch(filename, Constants.RedumpBiosPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.RedumpBiosPattern).Groups;
+				if (!Mappings.DatMaps["Redump"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped.");
+					return source;
+				}
+				source = "Redump";
+			}
+			else if (Regex.IsMatch(filename, Constants.TosecPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.TosecPattern).Groups;
+				if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+				{
+					// Handle special case mappings found only in TOSEC
+					fileinfo = Regex.Match(filename, Constants.TosecSpecialPatternA).Groups;
+
+					if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+					{
+						fileinfo = Regex.Match(filename, Constants.TosecSpecialPatternB).Groups;
+
+						if (!Mappings.DatMaps["TOSEC"].ContainsKey(fileinfo[1].Value))
+						{
+							_logger.Warning("The filename " + fileinfo[1].Value + " was matched as TOSEC but could not be mapped.");
+							return source;
+						}
+					}
+				}
+				source = "TOSEC";
+			}
+			else if (Regex.IsMatch(filename, Constants.TruripPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.TruripPattern).Groups;
+				if (!Mappings.DatMaps["TruRip"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as TruRip but could not be mapped.");
+					return source;
+				}
+				source = "trurip";
+			}
+			else if (Regex.IsMatch(filename, Constants.ZandroPattern))
+			{
+				source = "Zandro";
+			}
+			else if (Regex.IsMatch(filename, Constants.DefaultPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.DefaultPattern).Groups;
+				source = fileinfo[3].Value;
+			}
+			else if (Regex.IsMatch(filename, Constants.DefaultSpecialPattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.DefaultSpecialPattern).Groups;
+				source = fileinfo[3].Value;
+			}
+			else if (Regex.IsMatch(filename, Constants.MamePattern))
+			{
+				fileinfo = Regex.Match(filename, Constants.MamePattern).Groups;
+				if (!Mappings.DatMaps["MAME"].ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as MAME but could not be mapped.");
+					return source;
+				}
+				source = "MAME";
+			}
+
+			return source;
+		}
+	}
+}

SabreTools.Helper/Objects/OfflineMerge.cs

@@ -0,0 +1,502 @@
+using SabreTools.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace SabreTools
+{
+	public class OfflineMerge
+	{
+		// Instance variables
+		private string _currentAllMerged;
+		private string _currentMissingMerged;
+		private string _currentNewMerged;
+		private bool _fake;
+		private Logger _logger;
+
+		/// <summary>
+		/// Instantiate an OfflineMerge object
+		/// </summary>
+		/// <param name="currentAllMerged">Old-current DAT with merged and deduped values</param>
+		/// <param name="currentMissingMerged">Old-current missing DAT with merged and deduped values</param>
+		/// <param name="currentNewMerged">New-current DAT with merged and deduped values</param>
+		/// <param name="fake">True if all values should be replaced with default 0-byte values, false otherwise</param>
+		/// <param name="logger">Logger object for console and file output</param>
+		public OfflineMerge (string currentAllMerged, string currentMissingMerged, string currentNewMerged, bool fake, Logger logger)
+		{
+			_currentAllMerged = currentAllMerged;
+			_currentMissingMerged = currentMissingMerged;
+			_currentNewMerged = currentNewMerged;
+			_fake = fake;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Process the supplied inputs and create the four outputs
+		/// </summary>
+		/// <returns>True if the files were created properly, false otherwise</returns>
+		public bool Process()
+		{
+			// Check all of the files for validity and break if one doesn't exist
+			if (_currentAllMerged != "" && !File.Exists(_currentAllMerged))
+			{
+				return false;
+			}
+			if (_currentMissingMerged != "" && !File.Exists(_currentMissingMerged))
+			{
+				return false;
+			}
+			if (_currentNewMerged != "" && !File.Exists(_currentNewMerged))
+			{
+				return false;
+			}
+
+			// If we have all three DATs, then generate everything
+			if (_currentAllMerged != "" && _currentMissingMerged != "" && _currentNewMerged != "")
+			{
+				// First get the combination Dictionary of currentAllMerged and currentNewMerged
+				_logger.User("Adding Current and New Merged DATs to the dictionary");
+				Dat completeDats = new Dat();
+				completeDats = DatTools.Parse(_currentAllMerged, 0, 0, completeDats, _logger);
+				completeDats = DatTools.Parse(_currentNewMerged, 0, 0, completeDats, _logger);
+
+				// Now get Net New output dictionary [(currentNewMerged)-(currentAllMerged)]
+				_logger.User("Creating and populating Net New dictionary");
+				Dictionary<string, List<Rom>> netNew = new Dictionary<string, List<Rom>>();
+				foreach (string key in completeDats.Files.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(completeDats.Files[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
+						{
+							if (netNew.ContainsKey(key))
+							{
+								netNew[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								netNew.Add(key, temp);
+							}
+						}
+					}
+				}
+
+				// Now create the Unneeded dictionary [(currentAllMerged)-(currentNewMerged)]
+				_logger.User("Creating and populating Uneeded dictionary");
+				Dictionary<string, List<Rom>> unneeded = new Dictionary<string, List<Rom>>();
+				foreach (string key in completeDats.Files.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(completeDats.Files[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged)
+						{
+							if (unneeded.ContainsKey(key))
+							{
+								unneeded[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								unneeded.Add(key, temp);
+							}
+						}
+					}
+				}
+
+				// Now create the New Missing dictionary [(Net New)+(currentMissingMerged-(Unneeded))]
+				_logger.User("Creating and populating New Missing dictionary");
+				Dat midMissing = new Dat();
+				midMissing = DatTools.Parse(_currentMissingMerged, 0, 0, midMissing, _logger);
+				foreach (string key in unneeded.Keys)
+				{
+					if (midMissing.Files.ContainsKey(key))
+					{
+						midMissing.Files[key].AddRange(unneeded[key]);
+					}
+					else
+					{
+						midMissing.Files.Add(key, unneeded[key]);
+					}
+				}
+				Dictionary<string, List<Rom>> newMissing = new Dictionary<string, List<Rom>>();
+				foreach (string key in midMissing.Files.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(midMissing.Files[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentMissingMerged)
+						{
+							if (newMissing.ContainsKey(key))
+							{
+								newMissing[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								newMissing.Add(key, temp);
+							}
+						}
+					}
+				}
+				foreach (string key in netNew.Keys)
+				{
+					if (newMissing.ContainsKey(key))
+					{
+						newMissing[key].AddRange(netNew[key]);
+					}
+					else
+					{
+						newMissing.Add(key, netNew[key]);
+					}
+				}
+
+				// Now create the Have dictionary [(currentNewMerged)-(c)]
+				_logger.User("Creating and populating Have dictionary");
+				Dictionary<string, List<Rom>> midHave = new Dictionary<string, List<Rom>>();
+				foreach (string key in newMissing.Keys)
+				{
+					if (midHave.ContainsKey(key))
+					{
+						midHave[key].AddRange(newMissing[key]);
+					}
+					else
+					{
+						midHave.Add(key, newMissing[key]);
+					}
+				}
+				foreach (string key in completeDats.Files.Keys)
+				{
+					if (midHave.ContainsKey(key))
+					{
+						foreach (Rom rom in completeDats.Files[key])
+						{
+							if (rom.Metadata.System == _currentNewMerged)
+							{
+								midHave[key].Add(rom);
+							}
+						}
+					}
+					else
+					{
+						List<Rom> roms = new List<Rom>();
+						foreach (Rom rom in completeDats.Files[key])
+						{
+							if (rom.Metadata.System == _currentNewMerged)
+							{
+								roms.Add(rom);
+							}
+						}
+						midHave.Add(key, roms);
+					}
+				}
+				Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
+				foreach (string key in midHave.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(midHave[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
+						{
+							if (have.ContainsKey(key))
+							{
+								have[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								have.Add(key, temp);
+							}
+						}
+					}
+				}
+
+				// If we are supposed to replace everything in the output with default values, do so
+				if (_fake)
+				{
+					_logger.User("Replacing all hashes in Net New with 0-byte values");
+					List<string> keys = netNew.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = netNew[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						netNew[key] = temp;
+					}
+
+					_logger.User("Replacing all hashes in Unneeded with 0-byte values");
+					keys = unneeded.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = unneeded[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						unneeded[key] = temp;
+					}
+
+					_logger.User("Replacing all hashes in New Missing with 0-byte values");
+					keys = newMissing.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = newMissing[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						newMissing[key] = temp;
+					}
+
+					_logger.User("Replacing all hashes in Have with 0-byte values");
+					keys = have.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = have[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						have[key] = temp;
+					}
+				}
+
+				// Finally, output all of the files
+				Dat netNewData = new Dat
+				{
+					Name = "Net New",
+					Description = "Net New",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = netNew,
+				};
+				Dat unneededData = new Dat
+				{
+					Name = "Unneeded",
+					Description = "Unneeded",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = unneeded,
+				};
+				Dat newMissingData = new Dat
+				{
+					Name = "New Missing",
+					Description = "New Missing",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = newMissing,
+				};
+				Dat haveData = new Dat
+				{
+					Name = "Have",
+					Description = "Have",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = have,
+				};
+
+				DatTools.WriteDatfile(netNewData, "", _logger);
+				DatTools.WriteDatfile(unneededData, "", _logger);
+				DatTools.WriteDatfile(newMissingData, "", _logger);
+				DatTools.WriteDatfile(haveData, "", _logger);
+
+				return true;
+			}
+
+			// If we only have the old merged and missing, only generate Have
+			else if (_currentAllMerged != "" && _currentMissingMerged != "")
+			{
+				// Now create the Have dictionary [(currentAllMerged)-(currentMissingMerged)]
+				_logger.User("Creating and populating Have dictionary");
+				Dat midHave = new Dat();
+				midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger);
+				midHave = DatTools.Parse(_currentAllMerged, 0, 0, midHave, _logger);
+				Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
+				foreach (string key in midHave.Files.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(midHave.Files[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged)
+						{
+							if (have.ContainsKey(key))
+							{
+								have[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								have.Add(key, temp);
+							}
+						}
+					}
+				}
+
+				// If we are supposed to replace everything in the output with default values, do so
+				if (_fake)
+				{
+					_logger.User("Replacing all hashes in Have with 0-byte values");
+					List<string> keys = have.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = have[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						have[key] = temp;
+					}
+				}
+
+				Dat haveData = new Dat
+				{
+					Name = "Have",
+					Description = "Have",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = have,
+				};
+				DatTools.WriteDatfile(haveData, "", _logger);
+
+				return true;
+			}
+
+			// If we only have the new merged and missing, only generate Have
+			else if (_currentNewMerged != "" && _currentMissingMerged != "")
+			{
+				// Now create the Have dictionary [(currentNewMerged)-(currentMissingMerged)]
+				_logger.User("Creating and populating Have dictionary");
+				Dat midHave = new Dat();
+				midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger);
+				midHave = DatTools.Parse(_currentNewMerged, 0, 0, midHave, _logger);
+				Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
+				foreach (string key in midHave.Files.Keys)
+				{
+					List<Rom> templist = RomTools.Merge(midHave.Files[key], _logger);
+					foreach (Rom rom in templist)
+					{
+						if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
+						{
+							if (have.ContainsKey(key))
+							{
+								have[key].Add(rom);
+							}
+							else
+							{
+								List<Rom> temp = new List<Rom>();
+								temp.Add(rom);
+								have.Add(key, temp);
+							}
+						}
+					}
+				}
+
+				// If we are supposed to replace everything in the output with default values, do so
+				if (_fake)
+				{
+					_logger.User("Replacing all hashes in Have with 0-byte values");
+					List<string> keys = have.Keys.ToList();
+					foreach (string key in keys)
+					{
+						List<Rom> temp = new List<Rom>();
+						List<Rom> roms = have[key];
+						for (int i = 0; i < roms.Count; i++)
+						{
+							Rom rom = roms[i];
+							rom.HashData.Size = Constants.SizeZero;
+							rom.HashData.CRC = Constants.CRCZero;
+							rom.HashData.MD5 = Constants.MD5Zero;
+							rom.HashData.SHA1 = Constants.SHA1Zero;
+							temp.Add(rom);
+						}
+						have[key] = temp;
+					}
+				}
+
+				Dat haveData = new Dat
+				{
+					Name = "Have",
+					Description = "Have",
+					Version = "",
+					Date = DateTime.Now.ToString("yyyy-MM-dd"),
+					Category = "",
+					Author = "SabreTools",
+					ForcePacking = ForcePacking.None,
+					OutputFormat = OutputFormat.Xml,
+					MergeRoms = true,
+					Files = have,
+				};
+				DatTools.WriteDatfile(haveData, "", _logger);
+
+				return true;
+			}
+
+			return false;
+		}
+	}
+}

SabreTools.Helper/Objects/Split.cs

@@ -0,0 +1,455 @@
+using SabreTools.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace SabreTools
+{
+	public class Split
+	{
+		// Instance variables
+		private bool _hash;
+		private List<string> _extA;
+		private List<string> _extB;
+		private List<string> _inputs;
+		private string _outdir;
+		private static Logger _logger;
+
+		/// <summary>
+		/// Create a new Split object (extension split)
+		/// </summary>
+		/// <param name="filename">Filename of the DAT to split</param>
+		/// <param name="extA">List of extensions to split on (first DAT)</param>
+		/// <param name="extB">List of extensions to split on (second DAT)</param>
+		/// <param name="logger">Logger object for console and file writing</param>
+		public Split(List<string> inputs, List<string> extA, List<string> extB, string outdir, Logger logger)
+		{
+			_hash = false;
+			_inputs = inputs;
+			_extA = new List<string>();
+			foreach (string s in extA)
+			{
+				_extA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
+			}
+			_extB = new List<string>();
+			foreach (string s in extB)
+			{
+				_extB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
+			}
+			_outdir = outdir;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Create a new Split object (hash split)
+		/// </summary>
+		/// <param name="filename">Filename of the DAT to split</param>
+		/// <param name="logger">Logger object for console and file writing</param>
+		public Split(List<string> inputs, string outdir, Logger logger)
+		{
+			_hash = true;
+			_inputs = inputs;
+			_extA = new List<string>();
+			_extB = new List<string>();
+			_outdir = outdir;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Split a DAT based on filtering by 2 extensions
+		/// </summary>
+		/// <returns>True if DAT was split, false otherwise</returns>
+		public bool Process()
+		{
+			bool success = true;
+
+			// If it's empty, use the current folder
+			if (_outdir.Trim() == "")
+			{
+				_outdir = Environment.CurrentDirectory;
+			}
+
+			// If the output directory doesn't exist, create it
+			if (!Directory.Exists(_outdir))
+			{
+				Directory.CreateDirectory(_outdir);
+			}
+
+			// Loop over the inputs
+			foreach (string input in _inputs)
+			{
+				// If it's a file, run the proper split on the file
+				if (File.Exists(input))
+				{
+					if (_hash)
+					{
+						success &= SplitByHash(Path.GetFullPath(input), Path.GetDirectoryName(input));
+					}
+					else
+					{
+						success &= SplitByExt(Path.GetFullPath(input), Path.GetDirectoryName(input));
+					}
+				}
+				// If it's a directory, run the splits over the files within
+				else if (Directory.Exists(input))
+				{
+					foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
+					{
+						if (_hash)
+						{
+							success &= SplitByHash(Path.GetFullPath(file), (input.EndsWith(Path.DirectorySeparatorChar.ToString()) ? input : input + Path.DirectorySeparatorChar));
+						}
+						else
+						{
+							success &= SplitByExt(Path.GetFullPath(file), (input.EndsWith(Path.DirectorySeparatorChar.ToString()) ? input : input + Path.DirectorySeparatorChar));
+						}
+					}
+				}
+				// If file doesn't exist, error and return
+				else
+				{
+					_logger.Error("File or folder '" + input + "' doesn't exist");
+					return false;
+				}
+			}
+
+			return success;
+		}
+
+		/// <summary>
+		/// Split a DAT by best available hashes
+		/// </summary>
+		/// <param name="filename">Name of the file to be split</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		private bool SplitByHash(string filename, string basepath)
+		{
+			// Sanitize the basepath to be more predictable
+			basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
+
+			// Get the file data to be split
+			OutputFormat outputFormat = DatTools.GetOutputFormat(filename);
+			Dat datdata = new Dat();
+			datdata = DatTools.Parse(filename, 0, 0, datdata, _logger, true);
+
+			// Create each of the respective output DATs
+			_logger.User("Creating and populating new DATs");
+			Dat nodump = new Dat
+			{
+				FileName = datdata.FileName + " (Nodump)",
+				Name = datdata.Name + " (Nodump)",
+				Description = datdata.Description + " (Nodump)",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Header = datdata.Header,
+				Type = datdata.Type,
+				ForceMerging = datdata.ForceMerging,
+				ForceNodump = datdata.ForceNodump,
+				ForcePacking = datdata.ForcePacking,
+				OutputFormat = outputFormat,
+				MergeRoms = datdata.MergeRoms,
+				Files = new Dictionary<string, List<Rom>>(),
+			};
+			Dat sha1 = new Dat
+			{
+				FileName = datdata.FileName + " (SHA-1)",
+				Name = datdata.Name + " (SHA-1)",
+				Description = datdata.Description + " (SHA-1)",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Header = datdata.Header,
+				Type = datdata.Type,
+				ForceMerging = datdata.ForceMerging,
+				ForceNodump = datdata.ForceNodump,
+				ForcePacking = datdata.ForcePacking,
+				OutputFormat = outputFormat,
+				MergeRoms = datdata.MergeRoms,
+				Files = new Dictionary<string, List<Rom>>(),
+			};
+			Dat md5 = new Dat
+			{
+				FileName = datdata.FileName + " (MD5)",
+				Name = datdata.Name + " (MD5)",
+				Description = datdata.Description + " (MD5)",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Header = datdata.Header,
+				Type = datdata.Type,
+				ForceMerging = datdata.ForceMerging,
+				ForceNodump = datdata.ForceNodump,
+				ForcePacking = datdata.ForcePacking,
+				OutputFormat = outputFormat,
+				MergeRoms = datdata.MergeRoms,
+				Files = new Dictionary<string, List<Rom>>(),
+			};
+			Dat crc = new Dat
+			{
+				FileName = datdata.FileName + " (CRC)",
+				Name = datdata.Name + " (CRC)",
+				Description = datdata.Description + " (CRC)",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Header = datdata.Header,
+				Type = datdata.Type,
+				ForceMerging = datdata.ForceMerging,
+				ForceNodump = datdata.ForceNodump,
+				ForcePacking = datdata.ForcePacking,
+				OutputFormat = outputFormat,
+				MergeRoms = datdata.MergeRoms,
+				Files = new Dictionary<string, List<Rom>>(),
+			};
+
+			// Now populate each of the DAT objects in turn
+			List<string> keys = datdata.Files.Keys.ToList();
+			foreach (string key in keys)
+			{
+				List<Rom> roms = datdata.Files[key];
+				foreach (Rom rom in roms)
+				{
+					// If the file is a nodump
+					if (rom.Nodump)
+					{
+						if (nodump.Files.ContainsKey(key))
+						{
+							nodump.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							nodump.Files.Add(key, temp);
+						}
+					}
+					// If the file has a SHA-1
+					else if (rom.HashData.SHA1 != null && rom.HashData.SHA1 != "")
+					{
+						if (sha1.Files.ContainsKey(key))
+						{
+							sha1.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							sha1.Files.Add(key, temp);
+						}
+					}
+					// If the file has no SHA-1 but has an MD5
+					else if (rom.HashData.MD5 != null && rom.HashData.MD5 != "")
+					{
+						if (md5.Files.ContainsKey(key))
+						{
+							md5.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							md5.Files.Add(key, temp);
+						}
+					}
+					// All other cases
+					else
+					{
+						if (crc.Files.ContainsKey(key))
+						{
+							crc.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							crc.Files.Add(key, temp);
+						}
+					}
+				}
+			}
+
+			// Get the output directory
+			string outdir = "";
+			if (_outdir != "")
+			{
+				outdir = _outdir + Path.GetDirectoryName(filename).Remove(0, basepath.Length - 1);
+			}
+			else
+			{
+				outdir = Path.GetDirectoryName(filename);
+			}
+
+			// Now, output all of the files to the output directory
+			_logger.User("DAT information created, outputting new files");
+			bool success = true;
+			if (nodump.Files.Count > 0)
+			{
+				success &= DatTools.WriteDatfile(nodump, outdir, _logger);
+			}
+			if (sha1.Files.Count > 0)
+			{
+				success &= DatTools.WriteDatfile(sha1, outdir, _logger);
+			}
+			if (md5.Files.Count > 0)
+			{
+				success &= DatTools.WriteDatfile(md5, outdir, _logger);
+			}
+			if (crc.Files.Count > 0)
+			{
+				success &= DatTools.WriteDatfile(crc, outdir, _logger);
+			}
+
+			return success;
+		}
+
+		/// <summary>
+		/// Split a DAT by input extensions
+		/// </summary>
+		/// <param name="filename">Name of the file to be split</param>
+		/// <param name="basepath">Parent path for replacement</param>
+		/// <returns>True if split succeeded, false otherwise</returns>
+		private bool SplitByExt(string filename, string basepath)
+		{
+			// Load the current DAT to be processed
+			Dat datdata = new Dat();
+			datdata = DatTools.Parse(filename, 0, 0, datdata, _logger);
+
+			// Set all of the appropriate outputs for each of the subsets
+			OutputFormat outputFormat = DatTools.GetOutputFormat(filename);
+			Dat datdataA = new Dat
+			{
+				FileName = datdata.FileName + " (" + string.Join(",", _extA) + ")",
+				Name = datdata.Name + " (" + string.Join(",", _extA) + ")",
+				Description = datdata.Description + " (" + string.Join(",", _extA) + ")",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Files = new Dictionary<string, List<Rom>>(),
+				OutputFormat = outputFormat,
+			};
+			Dat datdataB = new Dat
+			{
+				FileName = datdata.FileName + " (" + string.Join(",", _extB) + ")",
+				Name = datdata.Name + " (" + string.Join(",", _extB) + ")",
+				Description = datdata.Description + " (" + string.Join(",", _extB) + ")",
+				Category = datdata.Category,
+				Version = datdata.Version,
+				Date = datdata.Date,
+				Author = datdata.Author,
+				Email = datdata.Email,
+				Homepage = datdata.Homepage,
+				Url = datdata.Url,
+				Comment = datdata.Comment,
+				Files = new Dictionary<string, List<Rom>>(),
+				OutputFormat = outputFormat,
+			};
+
+			// If roms is empty, return false
+			if (datdata.Files.Count == 0)
+			{
+				return false;
+			}
+
+			// Now separate the roms accordingly
+			foreach (string key in datdata.Files.Keys)
+			{
+				foreach (Rom rom in datdata.Files[key])
+				{
+					if (_extA.Contains(Path.GetExtension(rom.Name.ToUpperInvariant())))
+					{
+						if (datdataA.Files.ContainsKey(key))
+						{
+							datdataA.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							datdataA.Files.Add(key, temp);
+						}
+					}
+					else if (_extB.Contains(Path.GetExtension(rom.Name.ToUpperInvariant())))
+					{
+						if (datdataB.Files.ContainsKey(key))
+						{
+							datdataB.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							datdataB.Files.Add(key, temp);
+						}
+					}
+					else
+					{
+						if (datdataA.Files.ContainsKey(key))
+						{
+							datdataA.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							datdataA.Files.Add(key, temp);
+						}
+						if (datdataB.Files.ContainsKey(key))
+						{
+							datdataB.Files[key].Add(rom);
+						}
+						else
+						{
+							List<Rom> temp = new List<Rom>();
+							temp.Add(rom);
+							datdataB.Files.Add(key, temp);
+						}
+					}
+				}
+			}
+
+			// Get the output directory
+			string outdir = "";
+			if (_outdir != "")
+			{
+				outdir = _outdir + Path.GetDirectoryName(filename).Remove(0, basepath.Length - 1);
+			}
+			else
+			{
+				outdir = Path.GetDirectoryName(filename);
+			}
+
+			// Then write out both files
+			bool success = DatTools.WriteDatfile(datdataA, outdir, _logger);
+			success &= DatTools.WriteDatfile(datdataB, outdir, _logger);
+
+			return success;
+		}
+	}
+}

SabreTools.Helper/Objects/Stats.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+
+namespace SabreTools.Helper
+{
+	/// <summary>
+	/// Get statistics on one or more DAT files
+	/// </summary>
+	public class Stats
+	{
+		// Private instance variables
+		private List<String> _inputs;
+		private bool _single;
+		private Logger _logger;
+
+		/// <summary>
+		/// Create a new UncompressedSize object
+		/// </summary>
+		/// <param name="inputs">List of files and folders to parse</param>
+		/// <param name="single">True if single DAT stats are output, false otherwise</param>
+		/// <param name="logger">Logger object for file and console output</param>
+		public Stats(List<String> inputs, bool single, Logger logger)
+		{
+			_inputs = inputs;
+			_single = single;
+			_logger = logger;
+		}
+
+		/// <summary>
+		/// Output all requested statistics
+		/// </summary>
+		/// <returns>True if output succeeded, false otherwise</returns>
+		public bool Process()
+		{
+			// Init all total variables
+			long totalSize = 0;
+			long totalGame = 0;
+			long totalRom = 0;
+			long totalDisk = 0;
+			long totalCRC = 0;
+			long totalMD5 = 0;
+			long totalSHA1 = 0;
+			long totalNodump = 0;
+
+			/// Now process each of the input files
+			foreach (string filename in _inputs)
+			{
+				_logger.Log("Beginning stat collection for '" + filename + "'");
+				List<String> games = new List<String>();
+				Dat datdata = new Dat();
+				datdata = DatTools.Parse(filename, 0, 0, datdata, _logger);
+				SortedDictionary<string, List<Rom>> newroms = DatTools.BucketByGame(datdata.Files, false, true, _logger, false);
+
+				// Output single DAT stats (if asked)
+				if (_single)
+				{
+					_logger.User(@"\nFor file '" + filename + @"':
+--------------------------------------------------");
+					OutputStats(datdata, _logger);
+				}
+				else
+				{
+					_logger.User("Adding stats for file '" + filename + "'\n");
+				}
+
+				// Add single DAT stats to totals
+				totalSize += datdata.TotalSize;
+				totalGame += newroms.Count;
+				totalRom += datdata.RomCount;
+				totalDisk += datdata.DiskCount;
+				totalCRC += datdata.CRCCount;
+				totalMD5 += datdata.MD5Count;
+				totalSHA1 += datdata.SHA1Count;
+				totalNodump += datdata.NodumpCount;
+			}
+
+			// Output total DAT stats
+			if (!_single) { _logger.User(""); }
+			Dat totaldata = new Dat
+			{
+				TotalSize = totalSize,
+				RomCount = totalRom,
+				DiskCount = totalDisk,
+				CRCCount = totalCRC,
+				MD5Count = totalMD5,
+				SHA1Count = totalSHA1,
+				NodumpCount = totalNodump,
+			};
+			_logger.User(@"For ALL DATs found
+--------------------------------------------------");
+			OutputStats(totaldata, _logger, game:totalGame);
+			_logger.User(@"
+Please check the log folder if the stats scrolled offscreen");
+
+			return true;
+		}
+
+		/// <summary>
+		/// Output the stats in a human-readable format
+		/// </summary>
+		/// <param name="datdata">DatData object to read stats from</param>
+		/// <param name="logger">Logger object for file and console writing</param>
+		/// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param>
+		/// <param name="game">Number of games to use, -1 means recalculate games (default)</param>
+		public static void OutputStats(Dat datdata, Logger logger, bool recalculate = false, long game = -1)
+		{
+			if (recalculate)
+			{
+				// Wipe out any stats already there
+				datdata.RomCount = 0;
+				datdata.DiskCount = 0;
+				datdata.TotalSize = 0;
+				datdata.CRCCount = 0;
+				datdata.MD5Count = 0;
+				datdata.SHA1Count = 0;
+				datdata.NodumpCount = 0;
+
+				// Loop through and add
+				foreach (List<Rom> roms in datdata.Files.Values)
+				{
+					foreach (Rom rom in roms)
+					{
+						datdata.RomCount += (rom.Type == ItemType.Rom ? 1 : 0);
+						datdata.DiskCount += (rom.Type == ItemType.Disk ? 1 : 0);
+						datdata.TotalSize += (rom.Nodump ? 0 : rom.HashData.Size);
+						datdata.CRCCount += (String.IsNullOrEmpty(rom.HashData.CRC) ? 0 : 1);
+						datdata.MD5Count += (String.IsNullOrEmpty(rom.HashData.MD5) ? 0 : 1);
+						datdata.SHA1Count += (String.IsNullOrEmpty(rom.HashData.SHA1) ? 0 : 1);
+						datdata.NodumpCount += (rom.Nodump ? 1 : 0);
+					}
+				}
+			}
+
+			SortedDictionary<string, List<Rom>> newroms = DatTools.BucketByGame(datdata.Files, false, true, logger, false);
+			if (datdata.TotalSize < 0)
+			{
+				datdata.TotalSize = Int64.MaxValue + datdata.TotalSize;
+			}
+			logger.User("    Uncompressed size:       " + Style.GetBytesReadable(datdata.TotalSize) + @"
+    Games found:             " + (game == -1 ? newroms.Count : game) + @"
+    Roms found:              " + datdata.RomCount + @"
+    Disks found:             " + datdata.DiskCount + @"
+    Roms with CRC:           " + datdata.CRCCount + @"
+    Roms with MD5:           " + datdata.MD5Count + @"
+    Roms with SHA-1:         " + datdata.SHA1Count + @"
+    Roms with Nodump status: " + datdata.NodumpCount + @"
+");
+		}
+	}
+}