[ALL] Start work on restructuring based on hashes (will not compile)

This set of changes is the start of turning over to a Hash based system instead of a Rom/Dat based system. It's a long process but it will be worth it in the end.

SabreTools.Helper/Tools/ArchiveTools.cs

@@ -23,10 +23,10 @@ namespace SabreTools.Helper
 		{
 			string archiveFileName = Path.Combine(output, rom.Machine + ".zip");
 
-			System.IO.Compression.ZipArchive outarchive = null;
+			ZipArchive outarchive = null;
 			try
 			{
-				if (!System.IO.File.Exists(archiveFileName))
+				if (!File.Exists(archiveFileName))
 				{
 					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create);
 				}
@@ -35,7 +35,7 @@ namespace SabreTools.Helper
 					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update);
 				}
 
-				if (System.IO.File.Exists(input))
+				if (File.Exists(input))
 				{
 					if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null)
 					{
@@ -74,13 +74,13 @@ namespace SabreTools.Helper
 			string archiveFileName = Path.Combine(output, rom.Machine + ".zip");
 
 			// Delete an empty file first
-			if (System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
+			if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
 			{
-				System.IO.File.Delete(archiveFileName);
+				File.Delete(archiveFileName);
 			}
 
 			// Get if the file should be written out
-			bool newfile = System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
+			bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
 
 			using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
 				? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
@@ -88,7 +88,7 @@ namespace SabreTools.Helper
 			{
 				try
 				{
-					if (System.IO.File.Exists(input))
+					if (File.Exists(input))
 					{
 						archive.AddEntry(rom.Name, input);
 					}
@@ -105,10 +105,10 @@ namespace SabreTools.Helper
 				}
 			}
 
-			if (System.IO.File.Exists(archiveFileName + ".tmp"))
+			if (File.Exists(archiveFileName + ".tmp"))
 			{
-				System.IO.File.Delete(archiveFileName);
-				System.IO.File.Move(archiveFileName + ".tmp", archiveFileName);
+				File.Delete(archiveFileName);
+				File.Move(archiveFileName + ".tmp", archiveFileName);
 			}
 		}
 
@@ -172,7 +172,7 @@ namespace SabreTools.Helper
 			{
 				if (at == ArchiveType.SevenZip && sevenzip != ArchiveScanLevel.External)
 				{
-					sza = SevenZipArchive.Open(System.IO.File.OpenRead(input));
+					sza = SevenZipArchive.Open(File.OpenRead(input));
 					logger.Log("Found archive of type: " + at);
 
 					// Create the temp directory
@@ -189,9 +189,9 @@ namespace SabreTools.Helper
 					// Create the temp directory
 					Directory.CreateDirectory(tempdir);
 
-					using (FileStream itemstream = System.IO.File.OpenRead(input))
+					using (FileStream itemstream = File.OpenRead(input))
 					{
-						using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
+						using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
 						{
 							using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress))
 							{
@@ -203,7 +203,7 @@ namespace SabreTools.Helper
 				}
 				else
 				{
-					reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+					reader = ReaderFactory.Open(File.OpenRead(input));
 					logger.Log("Found archive of type: " + at);
 
 					if ((at == ArchiveType.Zip && zip != ArchiveScanLevel.External) ||
@@ -265,7 +265,7 @@ namespace SabreTools.Helper
 			IReader reader = null;
 			try
 			{
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+				reader = ReaderFactory.Open(File.OpenRead(input));
 
 				if (at == ArchiveType.Zip || at == ArchiveType.SevenZip || at == ArchiveType.Rar)
 				{
@@ -291,9 +291,9 @@ namespace SabreTools.Helper
 					// Dispose the original reader
 					reader.Dispose();
 
-					using(FileStream itemstream = System.IO.File.OpenRead(input))
+					using(FileStream itemstream = File.OpenRead(input))
 					{
-						using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
+						using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
 						{
 							using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress))
 							{
@@ -333,7 +333,7 @@ namespace SabreTools.Helper
 
 			// First get the archive types
 			ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger);
-			ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
+			ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
 
 			// If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return
 			if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive)
@@ -342,10 +342,10 @@ namespace SabreTools.Helper
 			}
 
 			IReader reader = null;
-			System.IO.Compression.ZipArchive outarchive = null;
+			ZipArchive outarchive = null;
 			try
 			{
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive));
+				reader = ReaderFactory.Open(File.OpenRead(inputArchive));
 
 				if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar)
 				{
@@ -354,7 +354,7 @@ namespace SabreTools.Helper
 						logger.Log("Current entry name: '" + reader.Entry.Key + "'");
 						if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName))
 						{
-							if (!System.IO.File.Exists(outputArchive))
+							if (!File.Exists(outputArchive))
 							{
 								outarchive = ZipFile.Open(outputArchive, ZipArchiveMode.Create);
 							}
@@ -365,7 +365,7 @@ namespace SabreTools.Helper
 
 							if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(destEntryName) == null)
 							{
-								System.IO.Compression.ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as System.IO.Compression.ZipArchiveEntry;
+								ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as ZipArchiveEntry;
 
 								using (Stream iaestream = iae.Open())
 								{
@@ -407,7 +407,7 @@ namespace SabreTools.Helper
 
 			// First get the archive types
 			ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger);
-			ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
+			ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
 
 			// If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return
 			if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive)
@@ -417,7 +417,7 @@ namespace SabreTools.Helper
 
 			try
 			{
-				using (IReader reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive)))
+				using (IReader reader = ReaderFactory.Open(File.OpenRead(inputArchive)))
 				{
 					if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar)
 					{
@@ -427,7 +427,7 @@ namespace SabreTools.Helper
 							if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName))
 							{
 								// Get if the file should be written out
-								bool newfile = System.IO.File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0;
+								bool newfile = File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0;
 
 								using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
 									? ArchiveFactory.Open(outputArchive, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
@@ -447,10 +447,10 @@ namespace SabreTools.Helper
 									}
 								}
 
-								if (System.IO.File.Exists(outputArchive + ".tmp"))
+								if (File.Exists(outputArchive + ".tmp"))
 								{
-									System.IO.File.Delete(outputArchive);
-									System.IO.File.Move(outputArchive + ".tmp", outputArchive);
+									File.Delete(outputArchive);
+									File.Move(outputArchive + ".tmp", outputArchive);
 								}
 
 								success = true;
@@ -512,7 +512,7 @@ namespace SabreTools.Helper
 				if (at == ArchiveType.GZip)
 				{
 					// Get the CRC and size from the file
-					using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+					using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 					{
 						br.BaseStream.Seek(-8, SeekOrigin.End);
 						byte[] headercrc = br.ReadBytes(4);
@@ -522,7 +522,7 @@ namespace SabreTools.Helper
 					}
 				}
 
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+				reader = ReaderFactory.Open(File.OpenRead(input));
 
 				if (at != ArchiveType.Tar)
 				{
@@ -594,7 +594,7 @@ namespace SabreTools.Helper
 			byte[] headermd5; // MD5
 			byte[] headercrc; // CRC
 			byte[] headersz; // Int64 size
-			using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 			{
 				header = br.ReadBytes(12);
 				headermd5 = br.ReadBytes(16);
@@ -650,7 +650,7 @@ namespace SabreTools.Helper
 		public static bool WriteTorrentGZ(string input, string outdir, bool romba, Logger logger)
 		{
 			// Check that the input file exists
-			if (!System.IO.File.Exists(input))
+			if (!File.Exists(input))
 			{
 				logger.Warning("File " + input + " does not exist!");
 				return false;
@@ -670,7 +670,7 @@ namespace SabreTools.Helper
 			// If it doesn't exist, create the output file and then write
 			string outfile = Path.Combine(outdir, rom.HashData.SHA1 + ".gz");
 			using (FileStream inputstream = new FileStream(input, FileMode.Open))
-			using (GZipStream output = new GZipStream(System.IO.File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress))
+			using (GZipStream output = new GZipStream(File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress))
 			{
 				inputstream.CopyTo(output);
 			}
@@ -678,7 +678,7 @@ namespace SabreTools.Helper
 			// Now that it's ready, inject the header info
 			using (BinaryWriter sw = new BinaryWriter(new MemoryStream()))
 			{
-				using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(outfile)))
+				using (BinaryReader br = new BinaryReader(File.OpenRead(outfile)))
 				{
 					// Write standard header and TGZ info
 					byte[] data = Constants.TorrentGZHeader
@@ -698,7 +698,7 @@ namespace SabreTools.Helper
 					}
 				}
 
-				using (BinaryWriter bw = new BinaryWriter(System.IO.File.Open(outfile, FileMode.Create)))
+				using (BinaryWriter bw = new BinaryWriter(File.Open(outfile, FileMode.Create)))
 				{
 					// Now write the new file over the original
 					sw.BaseStream.Seek(0, SeekOrigin.Begin);
@@ -719,12 +719,12 @@ namespace SabreTools.Helper
 
 				try
 				{
-					System.IO.File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile)));
+					File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile)));
 				}
 				catch (Exception ex)
 				{
 					logger.Warning(ex.ToString());
-					System.IO.File.Delete(outfile);
+					File.Delete(outfile);
 				}
 			}
 
@@ -754,7 +754,7 @@ namespace SabreTools.Helper
 			try
 			{
 				byte[] magic = new byte[8];
-				using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+				using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 				{
 					magic = br.ReadBytes(8);
 				}
@@ -851,5 +851,295 @@ namespace SabreTools.Helper
 				bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
 			return bytes;
 		}
+
+		#region Hash-to-Dat functions
+
+		// All things in this region are direct ports and do not take advantage of the multiple rom per hash that comes with the new system
+
+		/// <summary>
+		/// Copy a file to an output archive
+		/// </summary>
+		/// <param name="input">Input filename to be moved</param>
+		/// <param name="output">Output directory to build to</param>
+		/// <param name="hash">RomData representing the new information</param>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static void WriteToArchive(string input, string output, RomData rom, int machineIndex)
+		{
+			string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip");
+
+			ZipArchive outarchive = null;
+			try
+			{
+				if (!File.Exists(archiveFileName))
+				{
+					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create);
+				}
+				else
+				{
+					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update);
+				}
+
+				if (File.Exists(input))
+				{
+					if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null)
+					{
+						outarchive.CreateEntryFromFile(input, rom.Name, CompressionLevel.Optimal);
+					}
+				}
+				else if (Directory.Exists(input))
+				{
+					foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
+					{
+						if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(file) == null)
+						{
+							outarchive.CreateEntryFromFile(file, file, CompressionLevel.Optimal);
+						}
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				Console.WriteLine(ex);
+			}
+			finally
+			{
+				outarchive?.Dispose();
+			}
+		}
+
+		/// <summary>
+		/// Copy a file to an output archive using SharpCompress
+		/// </summary>
+		/// <param name="input">Input filename to be moved</param>
+		/// <param name="output">Output directory to build to</param>
+		/// <param name="rom">RomData representing the new information</param>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static void WriteToManagedArchive(string input, string output, RomData rom, int machineIndex)
+		{
+			string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip");
+
+			// Delete an empty file first
+			if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
+			{
+				File.Delete(archiveFileName);
+			}
+
+			// Get if the file should be written out
+			bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
+
+			using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
+				? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
+				: ArchiveFactory.Create(ArchiveType.Zip) as SharpCompress.Archive.Zip.ZipArchive))
+			{
+				try
+				{
+					if (File.Exists(input))
+					{
+						archive.AddEntry(rom.Name, input);
+					}
+					else if (Directory.Exists(input))
+					{
+						archive.AddAllFromDirectory(input, "*", SearchOption.AllDirectories);
+					}
+
+					archive.SaveTo(archiveFileName + ".tmp", CompressionType.Deflate);
+				}
+				catch (Exception)
+				{
+					// Don't log archive write errors
+				}
+			}
+
+			if (File.Exists(archiveFileName + ".tmp"))
+			{
+				File.Delete(archiveFileName);
+				File.Move(archiveFileName + ".tmp", archiveFileName);
+			}
+		}
+
+		/// <summary>
+		/// Generate a list of HashData objects from the header values in an archive
+		/// </summary>
+		/// <param name="input">Input file to get data from</param>
+		/// <param name="logger">Logger object for file and console output</param>
+		/// <returns>List of HashData objects representing the found data</returns>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static List<HashData> GetArchiveFileHashes(string input, Logger logger)
+		{
+			List<HashData> hashes = new List<HashData>();
+			string gamename = Path.GetFileNameWithoutExtension(input);
+
+			// First get the archive type
+			ArchiveType? at = GetCurrentArchiveType(input, logger);
+
+			// If we got back null, then it's not an archive, so we we return
+			if (at == null)
+			{
+				return hashes;
+			}
+
+			// If we got back GZip, try to get TGZ info first
+			else if (at == ArchiveType.GZip)
+			{
+				HashData possibleTgz = GetTorrentGZFileHash(input, logger);
+
+				// If it was, then add it to the outputs and continue
+				if (possibleTgz.Size != -1)
+				{
+					hashes.Add(possibleTgz);
+					return hashes;
+				}
+			}
+
+			IReader reader = null;
+			try
+			{
+				logger.Log("Found archive of type: " + at);
+				long size = 0;
+				byte[] crc = null;
+
+				// If we have a gzip file, get the crc directly
+				if (at == ArchiveType.GZip)
+				{
+					// Get the CRC and size from the file
+					using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+					{
+						br.BaseStream.Seek(-8, SeekOrigin.End);
+						crc = br.ReadBytes(4).Reverse().ToArray();
+						byte[] headersize = br.ReadBytes(4);
+						size = BitConverter.ToInt32(headersize.Reverse().ToArray(), 0);
+					}
+				}
+
+				reader = ReaderFactory.Open(File.OpenRead(input));
+
+				if (at != ArchiveType.Tar)
+				{
+					while (reader.MoveToNextEntry())
+					{
+						if (reader.Entry != null && !reader.Entry.IsDirectory)
+						{
+							logger.Log("Entry found: '" + reader.Entry.Key + "': "
+								+ (size == 0 ? reader.Entry.Size : size) + ", "
+								+ (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc));
+
+							MachineData tempmachine = new MachineData
+							{
+								Name = gamename,
+								Description = gamename,
+							};
+							RomData temprom = new RomData
+							{
+								Type = ItemType.Rom,
+								Name = reader.Entry.Key,
+								Machines = new List<MachineData>(),
+							};
+							temprom.Machines.Add(tempmachine);
+							HashData temphash = new HashData
+							{
+								Size = (size == 0 ? reader.Entry.Size : size),
+								CRC = (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc),
+								MD5 = null,
+								SHA1 = null,
+								Roms = new List<RomData>(),
+							};
+							temphash.Roms.Add(temprom);
+							hashes.Add(temphash);
+						}
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				logger.Error(ex.ToString());
+			}
+			finally
+			{
+				reader?.Dispose();
+			}
+
+			return hashes;
+		}
+
+		/// <summary>
+		/// Retrieve file information for a single torrent GZ file
+		/// </summary>
+		/// <param name="input">Filename to get information from</param>
+		/// <param name="logger">Logger object for file and console output</param>
+		/// <returns>Populated HashData object if success, empty one on error</returns>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static HashData GetTorrentGZFileHash(string input, Logger logger)
+		{
+			string datum = Path.GetFileName(input).ToLowerInvariant();
+			string sha1 = Path.GetFileNameWithoutExtension(input).ToLowerInvariant();
+			long filesize = new FileInfo(input).Length;
+
+			// Check if the name is the right length
+			if (!Regex.IsMatch(datum, @"^[0-9a-f]{40}\.gz"))
+			{
+				logger.Warning("Non SHA-1 filename found, skipping: '" + datum + "'");
+				return new HashData();
+			}
+
+			// Check if the file is at least the minimum length
+			if (filesize < 40 /* bytes */)
+			{
+				logger.Warning("Possibly corrupt file '" + input + "' with size " + Style.GetBytesReadable(filesize));
+				return new HashData();
+			}
+
+			// Get the Romba-specific header data
+			byte[] header; // Get preamble header for checking
+			byte[] headermd5; // MD5
+			byte[] headercrc; // CRC
+			byte[] headersz; // Int64 size
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+			{
+				header = br.ReadBytes(12);
+				headermd5 = br.ReadBytes(16);
+				headercrc = br.ReadBytes(4);
+				headersz = br.ReadBytes(8);
+			}
+
+			// If the header is not correct, return a blank rom
+			bool correct = true;
+			for (int i = 0; i < header.Length; i++)
+			{
+				correct &= (header[i] == Constants.TorrentGZHeader[i]);
+			}
+			if (!correct)
+			{
+				return new HashData();
+			}
+
+			// Now convert the size and get the right position
+			long extractedsize = (long)BitConverter.ToUInt64(headersz.Reverse().ToArray(), 0);
+
+			MachineData tempmachine = new MachineData
+			{
+				Name = sha1,
+				Description = sha1,
+			};
+			RomData temprom = new RomData
+			{
+				Type = ItemType.Rom,
+				Name = sha1,
+				Machines = new List<MachineData>(),
+			};
+			temprom.Machines.Add(tempmachine);
+			HashData temphash = new HashData
+			{
+				Size = extractedsize,
+				CRC = headercrc,
+				MD5 = headermd5,
+				SHA1 = StringToByteArray(sha1),
+				Roms = new List<RomData>(),
+			};
+			temphash.Roms.Add(temprom);
+
+			return temphash;
+		}
+
+		#endregion
 	}
 }

SabreTools.Helper/Tools/DBTools.cs

@@ -17,7 +17,7 @@ namespace SabreTools.Helper
 		public static void EnsureDatabase(string db, string connectionString)
 		{
 			// Make sure the file exists
-			if (!System.IO.File.Exists(db))
+			if (!File.Exists(db))
 			{
 				SqliteConnection.CreateFile(db);
 			}

SabreTools.Helper/Tools/Output.cs

@@ -85,7 +85,7 @@ namespace SabreTools.Helper
 
 			try
 			{
-				FileStream fs = System.IO.File.Create(outfile);
+				FileStream fs = File.Create(outfile);
 				StreamWriter sw = new StreamWriter(fs, Encoding.UTF8);
 
 				// Write out the header
@@ -619,7 +619,7 @@ namespace SabreTools.Helper
 			{
 				try
 				{
-					System.IO.File.Delete(file);
+					File.Delete(file);
 				}
 				catch { }
 			}
@@ -643,14 +643,14 @@ namespace SabreTools.Helper
 		public static void RemoveBytesFromFile(string input, string output, long bytesToRemoveFromHead, long bytesToRemoveFromTail)
 		{
 			// If any of the inputs are invalid, skip
-			if (!System.IO.File.Exists(input) || new FileInfo(input).Length <= (bytesToRemoveFromHead + bytesToRemoveFromTail))
+			if (!File.Exists(input) || new FileInfo(input).Length <= (bytesToRemoveFromHead + bytesToRemoveFromTail))
 			{
 				return;
 			}
 
 			// Read the input file and write to the fail
-			using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
-			using (BinaryWriter bw = new BinaryWriter(System.IO.File.OpenWrite(output)))
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+			using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output)))
 			{
 				int bufferSize = 1024;
 				long adjustedLength = br.BaseStream.Length - bytesToRemoveFromTail;
@@ -708,13 +708,13 @@ namespace SabreTools.Helper
 		public static void AppendBytesToFile(string input, string output, byte[] bytesToAddToHead, byte[] bytesToAddToTail)
 		{
 			// If any of the inputs are invalid, skip
-			if (!System.IO.File.Exists(input))
+			if (!File.Exists(input))
 			{
 				return;
 			}
 
-			using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
-			using (BinaryWriter bw = new BinaryWriter(System.IO.File.OpenWrite(output)))
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+			using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output)))
 			{
 				if (bytesToAddToHead.Count() > 0)
 				{
@@ -751,13 +751,13 @@ namespace SabreTools.Helper
 		/// <param name="output">Output filename</param>
 		public static void CopyFileToNewLocation(string input, string output)
 		{
-			if (System.IO.File.Exists(input) && !System.IO.File.Exists(output))
+			if (File.Exists(input) && !File.Exists(output))
 			{
 				if (!Directory.Exists(Path.GetDirectoryName(output)))
 				{
 					Directory.CreateDirectory(Path.GetDirectoryName(output));
 				}
-				System.IO.File.Copy(input, output);
+				File.Copy(input, output);
 			}
 		}
 		}

SabreTools.Helper/Tools/RomTools.cs

@@ -21,7 +21,7 @@ namespace SabreTools.Helper
 		public static Rom GetSingleFileInfo(string input, bool noMD5 = false, bool noSHA1 = false, long offset = 0)
 		{
 			// Add safeguard if file doesn't exist
-			if (!System.IO.File.Exists(input))
+			if (!File.Exists(input))
 			{
 				return new Rom();
 			}
@@ -44,7 +44,7 @@ namespace SabreTools.Helper
 				using (Crc32 crc = new Crc32())
 				using (MD5 md5 = MD5.Create())
 				using (SHA1 sha1 = SHA1.Create())
-				using (FileStream fs = System.IO.File.OpenRead(input))
+				using (FileStream fs = File.OpenRead(input))
 				{
 					// Seek to the starting position, if one is set
 					if (offset > 0)
@@ -186,6 +186,104 @@ namespace SabreTools.Helper
 			return outroms;
 		}
 
+		/// <summary>
+		/// Merge an arbitrary set of ROMs based on the supplied information
+		/// </summary>
+		/// <param name="inhashes">List of HashData objects representing the roms to be merged</param>
+		/// <param name="logger">Logger object for console and/or file output</param>
+		/// <returns>A List of HashData objects representing the merged roms</returns>
+		public static List<HashData> Merge(List<HashData> inhashes, Logger logger)
+		{
+			// Create output list
+			List<HashData> outroms = new List<HashData>();
+
+			// Check for null or blank roms first
+			if (inhashes == null || inhashes.Count == 0)
+			{
+				return outroms;
+			}
+
+			// Then deduplicate them by checking to see if data matches previous saved roms
+			foreach (HashData hash in inhashes)
+			{
+				// If it's a nodump, add and skip
+				if (hash.Roms[0].Nodump)
+				{
+					outroms.Add(hash);
+					continue;
+				}
+
+				// If it's the first rom in the list, don't touch it
+				if (outroms.Count != 0)
+				{
+					// Check if the rom is a duplicate
+					DupeType dupetype = DupeType.None;
+					HashData savedHash = new HashData();
+					int pos = -1;
+					for (int i = 0; i < outroms.Count; i++)
+					{
+						HashData lastrom = outroms[i];
+						RomData savedRom = savedHash.Roms[0];
+						MachineData savedMachine = savedRom.Machine;
+
+						// Get the duplicate status
+						dupetype = GetDuplicateStatus(hash, lastrom, logger);
+
+						// If it's a duplicate, skip adding it to the output but add any missing information
+						if (dupetype != DupeType.None)
+						{
+							savedHash = lastrom;
+							pos = i;
+
+							savedHash.CRC = (savedHash.CRC == null && hash.CRC != null ? hash.CRC : savedHash.CRC);
+							savedHash.MD5 = (savedHash.MD5 == null && hash.MD5 != null ? hash.MD5 : savedHash.MD5);
+							savedHash.SHA1 = (savedHash.SHA1 == null && hash.SHA1 != null ? hash.SHA1 : savedHash.SHA1);
+							savedRom.DupeType = dupetype;
+
+							// If the current system has a lower ID than the previous, set the system accordingly
+							if (hash.Roms[0].Machine.SystemID < savedMachine.SystemID)
+							{
+								savedMachine.SystemID = hash.Roms[0].Machine.SystemID;
+								savedMachine.System = hash.Roms[0].Machine.System;
+								savedMachine.Name = hash.Roms[0].Machine.Name;
+								savedRom.Name = hash.Roms[0].Name;
+							}
+
+							// If the current source has a lower ID than the previous, set the source accordingly
+							if (hash.Roms[0].Machine.SourceID < savedMachine.SourceID)
+							{
+								savedMachine.SourceID = hash.Roms[0].Machine.SourceID;
+								savedMachine.Source = hash.Roms[0].Machine.Source;
+								savedMachine.Name = hash.Roms[0].Machine.Name;
+								savedRom.Name = hash.Roms[0].Name;
+							}
+
+							break;
+						}
+					}
+
+					// If no duplicate is found, add it to the list
+					if (dupetype == DupeType.None)
+					{
+						outroms.Add(hash);
+					}
+					// Otherwise, if a new rom information is found, add that
+					else
+					{
+						outroms.RemoveAt(pos);
+						outroms.Insert(pos, savedHash);
+					}
+				}
+				else
+				{
+					outroms.Add(hash);
+				}
+			}
+
+			// Then return the result
+			return outroms;
+		}
+
 		/// <summary>
 		/// List all duplicates found in a DAT based on a rom
 		/// </summary>
@@ -348,5 +446,51 @@ namespace SabreTools.Helper
 
 			return hash;
 		}
+
+		/// <summary>
+		/// Clean a hash byte array and pad to the correct size
+		/// </summary>
+		/// <param name="hash">Hash byte array to sanitize</param>
+		/// <param name="padding">Amount of bytes to pad to</param>
+		/// <returns>Cleaned byte array</returns>
+		public static byte[] CleanHashData(byte[] hash, int padding)
+		{
+			// If we have a null hash or a <=0 padding, return the hash
+			if (hash == null || padding <= 0)
+			{
+				return hash;
+			}
+
+			// If we have a hash longer than the padding, trim and return
+			if (hash.Length > padding)
+			{
+				return hash.Take(padding).ToArray();
+			}
+
+			// If we have a hash of the correct length, return
+			if (hash.Length == padding)
+			{
+				return hash;
+			}
+
+			// Otherwise get the output byte array of the correct length
+			byte[] newhash = new byte[padding];
+
+			// Then write the proper number of empty bytes
+			int padNeeded = padding - hash.Length;
+			int index = 0;
+			for (index = 0; index < padNeeded; index++)
+			{
+				newhash[index] = 0x00;
+			}
+
+			// Now add the original hash
+			for (int i = 0; i < hash.Length; i++)
+			{
+				newhash[index + i] = hash[index];
+			}
+
+			return newhash;
+		}
 	}
 }