[ALL] Start work on restructuring based on hashes (will not compile)

This set of changes is the start of turning over to a Hash based system instead of a Rom/Dat based system. It's a long process but it will be worth it in the end.

SabreTools.Helper/Tools/ArchiveTools.cs

@@ -23,10 +23,10 @@ namespace SabreTools.Helper
 		{
 			string archiveFileName = Path.Combine(output, rom.Machine + ".zip");
 
-			System.IO.Compression.ZipArchive outarchive = null;
+			ZipArchive outarchive = null;
 			try
 			{
-				if (!System.IO.File.Exists(archiveFileName))
+				if (!File.Exists(archiveFileName))
 				{
 					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create);
 				}
@@ -35,7 +35,7 @@ namespace SabreTools.Helper
 					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update);
 				}
 
-				if (System.IO.File.Exists(input))
+				if (File.Exists(input))
 				{
 					if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null)
 					{
@@ -74,13 +74,13 @@ namespace SabreTools.Helper
 			string archiveFileName = Path.Combine(output, rom.Machine + ".zip");
 
 			// Delete an empty file first
-			if (System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
+			if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
 			{
-				System.IO.File.Delete(archiveFileName);
+				File.Delete(archiveFileName);
 			}
 
 			// Get if the file should be written out
-			bool newfile = System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
+			bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
 
 			using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
 				? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
@@ -88,7 +88,7 @@ namespace SabreTools.Helper
 			{
 				try
 				{
-					if (System.IO.File.Exists(input))
+					if (File.Exists(input))
 					{
 						archive.AddEntry(rom.Name, input);
 					}
@@ -105,10 +105,10 @@ namespace SabreTools.Helper
 				}
 			}
 
-			if (System.IO.File.Exists(archiveFileName + ".tmp"))
+			if (File.Exists(archiveFileName + ".tmp"))
 			{
-				System.IO.File.Delete(archiveFileName);
-				System.IO.File.Move(archiveFileName + ".tmp", archiveFileName);
+				File.Delete(archiveFileName);
+				File.Move(archiveFileName + ".tmp", archiveFileName);
 			}
 		}
 
@@ -172,7 +172,7 @@ namespace SabreTools.Helper
 			{
 				if (at == ArchiveType.SevenZip && sevenzip != ArchiveScanLevel.External)
 				{
-					sza = SevenZipArchive.Open(System.IO.File.OpenRead(input));
+					sza = SevenZipArchive.Open(File.OpenRead(input));
 					logger.Log("Found archive of type: " + at);
 
 					// Create the temp directory
@@ -189,9 +189,9 @@ namespace SabreTools.Helper
 					// Create the temp directory
 					Directory.CreateDirectory(tempdir);
 
-					using (FileStream itemstream = System.IO.File.OpenRead(input))
+					using (FileStream itemstream = File.OpenRead(input))
 					{
-						using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
+						using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
 						{
 							using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress))
 							{
@@ -203,7 +203,7 @@ namespace SabreTools.Helper
 				}
 				else
 				{
-					reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+					reader = ReaderFactory.Open(File.OpenRead(input));
 					logger.Log("Found archive of type: " + at);
 
 					if ((at == ArchiveType.Zip && zip != ArchiveScanLevel.External) ||
@@ -265,7 +265,7 @@ namespace SabreTools.Helper
 			IReader reader = null;
 			try
 			{
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+				reader = ReaderFactory.Open(File.OpenRead(input));
 
 				if (at == ArchiveType.Zip || at == ArchiveType.SevenZip || at == ArchiveType.Rar)
 				{
@@ -291,9 +291,9 @@ namespace SabreTools.Helper
 					// Dispose the original reader
 					reader.Dispose();
 
-					using(FileStream itemstream = System.IO.File.OpenRead(input))
+					using(FileStream itemstream = File.OpenRead(input))
 					{
-						using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
+						using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input))))
 						{
 							using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress))
 							{
@@ -333,7 +333,7 @@ namespace SabreTools.Helper
 
 			// First get the archive types
 			ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger);
-			ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
+			ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
 
 			// If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return
 			if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive)
@@ -342,10 +342,10 @@ namespace SabreTools.Helper
 			}
 
 			IReader reader = null;
-			System.IO.Compression.ZipArchive outarchive = null;
+			ZipArchive outarchive = null;
 			try
 			{
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive));
+				reader = ReaderFactory.Open(File.OpenRead(inputArchive));
 
 				if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar)
 				{
@@ -354,7 +354,7 @@ namespace SabreTools.Helper
 						logger.Log("Current entry name: '" + reader.Entry.Key + "'");
 						if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName))
 						{
-							if (!System.IO.File.Exists(outputArchive))
+							if (!File.Exists(outputArchive))
 							{
 								outarchive = ZipFile.Open(outputArchive, ZipArchiveMode.Create);
 							}
@@ -365,7 +365,7 @@ namespace SabreTools.Helper
 
 							if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(destEntryName) == null)
 							{
-								System.IO.Compression.ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as System.IO.Compression.ZipArchiveEntry;
+								ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as ZipArchiveEntry;
 
 								using (Stream iaestream = iae.Open())
 								{
@@ -407,7 +407,7 @@ namespace SabreTools.Helper
 
 			// First get the archive types
 			ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger);
-			ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
+			ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip);
 
 			// If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return
 			if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive)
@@ -417,7 +417,7 @@ namespace SabreTools.Helper
 
 			try
 			{
-				using (IReader reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive)))
+				using (IReader reader = ReaderFactory.Open(File.OpenRead(inputArchive)))
 				{
 					if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar)
 					{
@@ -427,7 +427,7 @@ namespace SabreTools.Helper
 							if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName))
 							{
 								// Get if the file should be written out
-								bool newfile = System.IO.File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0;
+								bool newfile = File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0;
 
 								using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
 									? ArchiveFactory.Open(outputArchive, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
@@ -447,10 +447,10 @@ namespace SabreTools.Helper
 									}
 								}
 
-								if (System.IO.File.Exists(outputArchive + ".tmp"))
+								if (File.Exists(outputArchive + ".tmp"))
 								{
-									System.IO.File.Delete(outputArchive);
-									System.IO.File.Move(outputArchive + ".tmp", outputArchive);
+									File.Delete(outputArchive);
+									File.Move(outputArchive + ".tmp", outputArchive);
 								}
 
 								success = true;
@@ -512,7 +512,7 @@ namespace SabreTools.Helper
 				if (at == ArchiveType.GZip)
 				{
 					// Get the CRC and size from the file
-					using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+					using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 					{
 						br.BaseStream.Seek(-8, SeekOrigin.End);
 						byte[] headercrc = br.ReadBytes(4);
@@ -522,7 +522,7 @@ namespace SabreTools.Helper
 					}
 				}
 
-				reader = ReaderFactory.Open(System.IO.File.OpenRead(input));
+				reader = ReaderFactory.Open(File.OpenRead(input));
 
 				if (at != ArchiveType.Tar)
 				{
@@ -594,7 +594,7 @@ namespace SabreTools.Helper
 			byte[] headermd5; // MD5
 			byte[] headercrc; // CRC
 			byte[] headersz; // Int64 size
-			using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 			{
 				header = br.ReadBytes(12);
 				headermd5 = br.ReadBytes(16);
@@ -650,7 +650,7 @@ namespace SabreTools.Helper
 		public static bool WriteTorrentGZ(string input, string outdir, bool romba, Logger logger)
 		{
 			// Check that the input file exists
-			if (!System.IO.File.Exists(input))
+			if (!File.Exists(input))
 			{
 				logger.Warning("File " + input + " does not exist!");
 				return false;
@@ -670,7 +670,7 @@ namespace SabreTools.Helper
 			// If it doesn't exist, create the output file and then write
 			string outfile = Path.Combine(outdir, rom.HashData.SHA1 + ".gz");
 			using (FileStream inputstream = new FileStream(input, FileMode.Open))
-			using (GZipStream output = new GZipStream(System.IO.File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress))
+			using (GZipStream output = new GZipStream(File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress))
 			{
 				inputstream.CopyTo(output);
 			}
@@ -678,7 +678,7 @@ namespace SabreTools.Helper
 			// Now that it's ready, inject the header info
 			using (BinaryWriter sw = new BinaryWriter(new MemoryStream()))
 			{
-				using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(outfile)))
+				using (BinaryReader br = new BinaryReader(File.OpenRead(outfile)))
 				{
 					// Write standard header and TGZ info
 					byte[] data = Constants.TorrentGZHeader
@@ -698,7 +698,7 @@ namespace SabreTools.Helper
 					}
 				}
 
-				using (BinaryWriter bw = new BinaryWriter(System.IO.File.Open(outfile, FileMode.Create)))
+				using (BinaryWriter bw = new BinaryWriter(File.Open(outfile, FileMode.Create)))
 				{
 					// Now write the new file over the original
 					sw.BaseStream.Seek(0, SeekOrigin.Begin);
@@ -719,12 +719,12 @@ namespace SabreTools.Helper
 
 				try
 				{
-					System.IO.File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile)));
+					File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile)));
 				}
 				catch (Exception ex)
 				{
 					logger.Warning(ex.ToString());
-					System.IO.File.Delete(outfile);
+					File.Delete(outfile);
 				}
 			}
 
@@ -754,7 +754,7 @@ namespace SabreTools.Helper
 			try
 			{
 				byte[] magic = new byte[8];
-				using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)))
+				using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
 				{
 					magic = br.ReadBytes(8);
 				}
@@ -851,5 +851,295 @@ namespace SabreTools.Helper
 				bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
 			return bytes;
 		}
+
+		#region Hash-to-Dat functions
+
+		// All things in this region are direct ports and do not take advantage of the multiple rom per hash that comes with the new system
+
+		/// <summary>
+		/// Copy a file to an output archive
+		/// </summary>
+		/// <param name="input">Input filename to be moved</param>
+		/// <param name="output">Output directory to build to</param>
+		/// <param name="hash">RomData representing the new information</param>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static void WriteToArchive(string input, string output, RomData rom, int machineIndex)
+		{
+			string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip");
+
+			ZipArchive outarchive = null;
+			try
+			{
+				if (!File.Exists(archiveFileName))
+				{
+					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create);
+				}
+				else
+				{
+					outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update);
+				}
+
+				if (File.Exists(input))
+				{
+					if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null)
+					{
+						outarchive.CreateEntryFromFile(input, rom.Name, CompressionLevel.Optimal);
+					}
+				}
+				else if (Directory.Exists(input))
+				{
+					foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
+					{
+						if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(file) == null)
+						{
+							outarchive.CreateEntryFromFile(file, file, CompressionLevel.Optimal);
+						}
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				Console.WriteLine(ex);
+			}
+			finally
+			{
+				outarchive?.Dispose();
+			}
+		}
+
+		/// <summary>
+		/// Copy a file to an output archive using SharpCompress
+		/// </summary>
+		/// <param name="input">Input filename to be moved</param>
+		/// <param name="output">Output directory to build to</param>
+		/// <param name="rom">RomData representing the new information</param>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static void WriteToManagedArchive(string input, string output, RomData rom, int machineIndex)
+		{
+			string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip");
+
+			// Delete an empty file first
+			if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0)
+			{
+				File.Delete(archiveFileName);
+			}
+
+			// Get if the file should be written out
+			bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0;
+
+			using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile
+				? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive
+				: ArchiveFactory.Create(ArchiveType.Zip) as SharpCompress.Archive.Zip.ZipArchive))
+			{
+				try
+				{
+					if (File.Exists(input))
+					{
+						archive.AddEntry(rom.Name, input);
+					}
+					else if (Directory.Exists(input))
+					{
+						archive.AddAllFromDirectory(input, "*", SearchOption.AllDirectories);
+					}
+
+					archive.SaveTo(archiveFileName + ".tmp", CompressionType.Deflate);
+				}
+				catch (Exception)
+				{
+					// Don't log archive write errors
+				}
+			}
+
+			if (File.Exists(archiveFileName + ".tmp"))
+			{
+				File.Delete(archiveFileName);
+				File.Move(archiveFileName + ".tmp", archiveFileName);
+			}
+		}
+
+		/// <summary>
+		/// Generate a list of HashData objects from the header values in an archive
+		/// </summary>
+		/// <param name="input">Input file to get data from</param>
+		/// <param name="logger">Logger object for file and console output</param>
+		/// <returns>List of HashData objects representing the found data</returns>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static List<HashData> GetArchiveFileHashes(string input, Logger logger)
+		{
+			List<HashData> hashes = new List<HashData>();
+			string gamename = Path.GetFileNameWithoutExtension(input);
+
+			// First get the archive type
+			ArchiveType? at = GetCurrentArchiveType(input, logger);
+
+			// If we got back null, then it's not an archive, so we we return
+			if (at == null)
+			{
+				return hashes;
+			}
+
+			// If we got back GZip, try to get TGZ info first
+			else if (at == ArchiveType.GZip)
+			{
+				HashData possibleTgz = GetTorrentGZFileHash(input, logger);
+
+				// If it was, then add it to the outputs and continue
+				if (possibleTgz.Size != -1)
+				{
+					hashes.Add(possibleTgz);
+					return hashes;
+				}
+			}
+
+			IReader reader = null;
+			try
+			{
+				logger.Log("Found archive of type: " + at);
+				long size = 0;
+				byte[] crc = null;
+
+				// If we have a gzip file, get the crc directly
+				if (at == ArchiveType.GZip)
+				{
+					// Get the CRC and size from the file
+					using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+					{
+						br.BaseStream.Seek(-8, SeekOrigin.End);
+						crc = br.ReadBytes(4).Reverse().ToArray();
+						byte[] headersize = br.ReadBytes(4);
+						size = BitConverter.ToInt32(headersize.Reverse().ToArray(), 0);
+					}
+				}
+
+				reader = ReaderFactory.Open(File.OpenRead(input));
+
+				if (at != ArchiveType.Tar)
+				{
+					while (reader.MoveToNextEntry())
+					{
+						if (reader.Entry != null && !reader.Entry.IsDirectory)
+						{
+							logger.Log("Entry found: '" + reader.Entry.Key + "': "
+								+ (size == 0 ? reader.Entry.Size : size) + ", "
+								+ (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc));
+
+							MachineData tempmachine = new MachineData
+							{
+								Name = gamename,
+								Description = gamename,
+							};
+							RomData temprom = new RomData
+							{
+								Type = ItemType.Rom,
+								Name = reader.Entry.Key,
+								Machines = new List<MachineData>(),
+							};
+							temprom.Machines.Add(tempmachine);
+							HashData temphash = new HashData
+							{
+								Size = (size == 0 ? reader.Entry.Size : size),
+								CRC = (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc),
+								MD5 = null,
+								SHA1 = null,
+								Roms = new List<RomData>(),
+							};
+							temphash.Roms.Add(temprom);
+							hashes.Add(temphash);
+						}
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				logger.Error(ex.ToString());
+			}
+			finally
+			{
+				reader?.Dispose();
+			}
+
+			return hashes;
+		}
+
+		/// <summary>
+		/// Retrieve file information for a single torrent GZ file
+		/// </summary>
+		/// <param name="input">Filename to get information from</param>
+		/// <param name="logger">Logger object for file and console output</param>
+		/// <returns>Populated HashData object if success, empty one on error</returns>
+		/// <remarks>This uses the new system that is not implemented anywhere yet</remarks>
+		public static HashData GetTorrentGZFileHash(string input, Logger logger)
+		{
+			string datum = Path.GetFileName(input).ToLowerInvariant();
+			string sha1 = Path.GetFileNameWithoutExtension(input).ToLowerInvariant();
+			long filesize = new FileInfo(input).Length;
+
+			// Check if the name is the right length
+			if (!Regex.IsMatch(datum, @"^[0-9a-f]{40}\.gz"))
+			{
+				logger.Warning("Non SHA-1 filename found, skipping: '" + datum + "'");
+				return new HashData();
+			}
+
+			// Check if the file is at least the minimum length
+			if (filesize < 40 /* bytes */)
+			{
+				logger.Warning("Possibly corrupt file '" + input + "' with size " + Style.GetBytesReadable(filesize));
+				return new HashData();
+			}
+
+			// Get the Romba-specific header data
+			byte[] header; // Get preamble header for checking
+			byte[] headermd5; // MD5
+			byte[] headercrc; // CRC
+			byte[] headersz; // Int64 size
+			using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
+			{
+				header = br.ReadBytes(12);
+				headermd5 = br.ReadBytes(16);
+				headercrc = br.ReadBytes(4);
+				headersz = br.ReadBytes(8);
+			}
+
+			// If the header is not correct, return a blank rom
+			bool correct = true;
+			for (int i = 0; i < header.Length; i++)
+			{
+				correct &= (header[i] == Constants.TorrentGZHeader[i]);
+			}
+			if (!correct)
+			{
+				return new HashData();
+			}
+
+			// Now convert the size and get the right position
+			long extractedsize = (long)BitConverter.ToUInt64(headersz.Reverse().ToArray(), 0);
+
+			MachineData tempmachine = new MachineData
+			{
+				Name = sha1,
+				Description = sha1,
+			};
+			RomData temprom = new RomData
+			{
+				Type = ItemType.Rom,
+				Name = sha1,
+				Machines = new List<MachineData>(),
+			};
+			temprom.Machines.Add(tempmachine);
+			HashData temphash = new HashData
+			{
+				Size = extractedsize,
+				CRC = headercrc,
+				MD5 = headermd5,
+				SHA1 = StringToByteArray(sha1),
+				Roms = new List<RomData>(),
+			};
+			temphash.Roms.Add(temprom);
+
+			return temphash;
+		}
+
+		#endregion
 	}
 }