diff --git a/Deheader/Headerer.cs b/Deheader/Headerer.cs index 94f896de..b3ce7a54 100644 --- a/Deheader/Headerer.cs +++ b/Deheader/Headerer.cs @@ -92,7 +92,7 @@ namespace SabreTools flag = false; break; default: - if (System.IO.File.Exists(temparg) || Directory.Exists(temparg)) + if (File.Exists(temparg) || Directory.Exists(temparg)) { input = temparg; } @@ -164,7 +164,7 @@ namespace SabreTools if (_flag) { // If it's a single file, just check it - if (System.IO.File.Exists(_input)) + if (File.Exists(_input)) { DetectSkipperAndTransform(_input); } @@ -183,7 +183,7 @@ namespace SabreTools else { // If it's a single file, just check it - if (System.IO.File.Exists(_input)) + if (File.Exists(_input)) { ReplaceHeader(_input); } @@ -224,7 +224,7 @@ namespace SabreTools // Now take care of the header and new output file string hstr = string.Empty; - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(file))) + using (BinaryReader br = new BinaryReader(File.OpenRead(file))) { // Extract the header as a string for the database byte[] hbin = br.ReadBytes(headerSize); @@ -248,13 +248,13 @@ namespace SabreTools Skippers.TransformFile(file, newfile, rule, _logger); // If the output file doesn't exist, return false - if (!System.IO.File.Exists(newfile)) + if (!File.Exists(newfile)) { return false; } // Now add the information to the database if it's not already there - Helper.Rom rom = RomTools.GetSingleFileInfo(newfile); + Rom rom = RomTools.GetSingleFileInfo(newfile); AddHeaderToDatabase(hstr, rom.HashData.SHA1, type); } @@ -309,7 +309,7 @@ namespace SabreTools public bool ReplaceHeader(string file) { // First, get the SHA-1 hash of the file - Helper.Rom rom = RomTools.GetSingleFileInfo(file); + Rom rom = RomTools.GetSingleFileInfo(file); // Then try to pull the corresponding headers from the database string header = ""; diff --git a/SabreTools.Helper/Data/Constants.cs b/SabreTools.Helper/Data/Constants.cs index ce2020bb..ea2dc135 100644 --- a/SabreTools.Helper/Data/Constants.cs +++ b/SabreTools.Helper/Data/Constants.cs @@ -14,16 +14,22 @@ namespace SabreTools.Helper public static long SizeZero = 0; public static string CRCZero = "00000000"; + public static byte[] CRCZeroBytes = new byte[] { 0x00, 0x00, 0x00, 0x00 }; public static string MD5Zero = "d41d8cd98f00b204e9800998ecf8427e"; + public static byte[] MD5ZeroBytes = new byte[] { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e }; public static string SHA1Zero = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + public static byte[] SHA1ZeroBytes = new byte[] { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09 }; #endregion #region Hash string length constants public static int CRCLength = 8; + public static int CRCBytesLength = 4; public static int MD5Length = 32; + public static int MD5BytesLength = 16; public static int SHA1Length = 40; + public static int SHA1BytesLength = 20; #endregion diff --git a/SabreTools.Helper/Data/Structs.cs b/SabreTools.Helper/Data/Structs.cs index d88fd08e..36f7d87f 100644 --- a/SabreTools.Helper/Data/Structs.cs +++ b/SabreTools.Helper/Data/Structs.cs @@ -28,9 +28,9 @@ namespace SabreTools.Helper public struct HashData { public long Size; - public string CRC; - public string MD5; - public string SHA1; + public byte[] CRC; + public byte[] MD5; + public byte[] SHA1; public List Roms; } @@ -43,7 +43,8 @@ namespace SabreTools.Helper public ItemType Type; public bool Nodump; public string Date; - public List Machines; + public DupeType DupeType; + public MachineData Machine; } /// @@ -51,6 +52,7 @@ namespace SabreTools.Helper /// public struct MachineData { + // Data specific to Machine/Game public string Name; public string Comment; public string Description; @@ -64,6 +66,12 @@ namespace SabreTools.Helper public string Board; public string RebuildTo; public bool TorrentZipped; + + // Data specific to the source of the Machine/Game + public int SystemID; + public string System; + public int SourceID; + public string Source; } /// @@ -91,6 +99,7 @@ namespace SabreTools.Helper public ForcePacking ForcePacking; public OutputFormat OutputFormat; public bool MergeRoms; + public List Hashes; // Data specific to the Miss DAT type public bool UseGame; diff --git a/SabreTools.Helper/Logger.cs b/SabreTools.Helper/Logger.cs index a185de24..56bcec71 100644 --- a/SabreTools.Helper/Logger.cs +++ b/SabreTools.Helper/Logger.cs @@ -70,7 +70,7 @@ namespace SabreTools.Helper try { - _log = new StreamWriter(System.IO.File.Open(_basepath + _filename, FileMode.OpenOrCreate | FileMode.Append)); + _log = new StreamWriter(File.Open(_basepath + _filename, FileMode.OpenOrCreate | FileMode.Append)); _log.WriteLine("Logging started " + DateTime.Now); _log.Flush(); } diff --git a/SabreTools.Helper/Mappings.cs b/SabreTools.Helper/Mappings.cs index f9d0cf03..69fc54c9 100644 --- a/SabreTools.Helper/Mappings.cs +++ b/SabreTools.Helper/Mappings.cs @@ -58,7 +58,7 @@ namespace SabreTools.Helper XmlDocument doc = new XmlDocument(); try { - doc.LoadXml(System.IO.File.ReadAllText(Path.Combine(_remappersPath, mapping + ".xml"))); + doc.LoadXml(File.ReadAllText(Path.Combine(_remappersPath, mapping + ".xml"))); } catch (XmlException ex) { diff --git a/SabreTools.Helper/Skippers.cs b/SabreTools.Helper/Skippers.cs index e5800c3a..523c830b 100644 --- a/SabreTools.Helper/Skippers.cs +++ b/SabreTools.Helper/Skippers.cs @@ -74,7 +74,7 @@ namespace SabreTools.Helper Rules = new List(), }; - if (!System.IO.File.Exists(filename)) + if (!File.Exists(filename)) { return skipper; } @@ -319,7 +319,7 @@ namespace SabreTools.Helper SkipperRule skipperRule = new SkipperRule(); // If the file doesn't exist, return a blank skipper rule - if (!System.IO.File.Exists(input)) + if (!File.Exists(input)) { logger.Error("The file '" + input + "' does not exist so it cannot be tested"); return skipperRule; @@ -332,7 +332,7 @@ namespace SabreTools.Helper if (String.IsNullOrEmpty(skippername) || (!String.IsNullOrEmpty(skipper.Name) && skippername.ToLowerInvariant() == skipper.Name.ToLowerInvariant())) { // Loop through the rules until one is found that works - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) { foreach (SkipperRule rule in skipper.Rules) { @@ -495,7 +495,7 @@ namespace SabreTools.Helper bool success = true; // If the input file doesn't exist, fail - if (!System.IO.File.Exists(input)) + if (!File.Exists(input)) { logger.Error("I'm sorry but '" + input + "' doesn't exist!"); return false; @@ -521,8 +521,8 @@ namespace SabreTools.Helper try { logger.User("Applying found rule to file '" + input + "'"); - using (BinaryWriter bw = new BinaryWriter(System.IO.File.OpenWrite(output))) - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) + using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) { // Seek to the beginning offset if (rule.StartOffset == null) @@ -619,7 +619,7 @@ namespace SabreTools.Helper { try { - System.IO.File.Delete(output); + File.Delete(output); } catch { @@ -663,7 +663,7 @@ namespace SabreTools.Helper XmlDocument doc = new XmlDocument(); try { - doc.LoadXml(System.IO.File.ReadAllText(System.IO.Path.Combine(Path, skipper + ".xml"))); + doc.LoadXml(File.ReadAllText(System.IO.Path.Combine(Path, skipper + ".xml"))); } catch (XmlException ex) { @@ -726,7 +726,7 @@ namespace SabreTools.Helper public static HeaderType GetFileHeaderType(string input, out int hs, Logger logger) { // Open the file in read mode - BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input)); + BinaryReader br = new BinaryReader(File.OpenRead(input)); // Extract the first 1024 bytes of the file byte[] hbin = br.ReadBytes(1024); diff --git a/SabreTools.Helper/Tools/ArchiveTools.cs b/SabreTools.Helper/Tools/ArchiveTools.cs index 6157e81c..d639a8a9 100644 --- a/SabreTools.Helper/Tools/ArchiveTools.cs +++ b/SabreTools.Helper/Tools/ArchiveTools.cs @@ -23,10 +23,10 @@ namespace SabreTools.Helper { string archiveFileName = Path.Combine(output, rom.Machine + ".zip"); - System.IO.Compression.ZipArchive outarchive = null; + ZipArchive outarchive = null; try { - if (!System.IO.File.Exists(archiveFileName)) + if (!File.Exists(archiveFileName)) { outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create); } @@ -35,7 +35,7 @@ namespace SabreTools.Helper outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update); } - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null) { @@ -74,13 +74,13 @@ namespace SabreTools.Helper string archiveFileName = Path.Combine(output, rom.Machine + ".zip"); // Delete an empty file first - if (System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0) + if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0) { - System.IO.File.Delete(archiveFileName); + File.Delete(archiveFileName); } // Get if the file should be written out - bool newfile = System.IO.File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0; + bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0; using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile ? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive @@ -88,7 +88,7 @@ namespace SabreTools.Helper { try { - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { archive.AddEntry(rom.Name, input); } @@ -105,10 +105,10 @@ namespace SabreTools.Helper } } - if (System.IO.File.Exists(archiveFileName + ".tmp")) + if (File.Exists(archiveFileName + ".tmp")) { - System.IO.File.Delete(archiveFileName); - System.IO.File.Move(archiveFileName + ".tmp", archiveFileName); + File.Delete(archiveFileName); + File.Move(archiveFileName + ".tmp", archiveFileName); } } @@ -172,7 +172,7 @@ namespace SabreTools.Helper { if (at == ArchiveType.SevenZip && sevenzip != ArchiveScanLevel.External) { - sza = SevenZipArchive.Open(System.IO.File.OpenRead(input)); + sza = SevenZipArchive.Open(File.OpenRead(input)); logger.Log("Found archive of type: " + at); // Create the temp directory @@ -189,9 +189,9 @@ namespace SabreTools.Helper // Create the temp directory Directory.CreateDirectory(tempdir); - using (FileStream itemstream = System.IO.File.OpenRead(input)) + using (FileStream itemstream = File.OpenRead(input)) { - using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input)))) + using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input)))) { using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress)) { @@ -203,7 +203,7 @@ namespace SabreTools.Helper } else { - reader = ReaderFactory.Open(System.IO.File.OpenRead(input)); + reader = ReaderFactory.Open(File.OpenRead(input)); logger.Log("Found archive of type: " + at); if ((at == ArchiveType.Zip && zip != ArchiveScanLevel.External) || @@ -265,7 +265,7 @@ namespace SabreTools.Helper IReader reader = null; try { - reader = ReaderFactory.Open(System.IO.File.OpenRead(input)); + reader = ReaderFactory.Open(File.OpenRead(input)); if (at == ArchiveType.Zip || at == ArchiveType.SevenZip || at == ArchiveType.Rar) { @@ -291,9 +291,9 @@ namespace SabreTools.Helper // Dispose the original reader reader.Dispose(); - using(FileStream itemstream = System.IO.File.OpenRead(input)) + using(FileStream itemstream = File.OpenRead(input)) { - using (FileStream outstream = System.IO.File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input)))) + using (FileStream outstream = File.Create(Path.Combine(tempdir, Path.GetFileNameWithoutExtension(input)))) { using (GZipStream gzstream = new GZipStream(itemstream, CompressionMode.Decompress)) { @@ -333,7 +333,7 @@ namespace SabreTools.Helper // First get the archive types ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger); - ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip); + ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip); // If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive) @@ -342,10 +342,10 @@ namespace SabreTools.Helper } IReader reader = null; - System.IO.Compression.ZipArchive outarchive = null; + ZipArchive outarchive = null; try { - reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive)); + reader = ReaderFactory.Open(File.OpenRead(inputArchive)); if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar) { @@ -354,7 +354,7 @@ namespace SabreTools.Helper logger.Log("Current entry name: '" + reader.Entry.Key + "'"); if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName)) { - if (!System.IO.File.Exists(outputArchive)) + if (!File.Exists(outputArchive)) { outarchive = ZipFile.Open(outputArchive, ZipArchiveMode.Create); } @@ -365,7 +365,7 @@ namespace SabreTools.Helper if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(destEntryName) == null) { - System.IO.Compression.ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as System.IO.Compression.ZipArchiveEntry; + ZipArchiveEntry iae = outarchive.CreateEntry(destEntryName, CompressionLevel.Optimal) as ZipArchiveEntry; using (Stream iaestream = iae.Open()) { @@ -407,7 +407,7 @@ namespace SabreTools.Helper // First get the archive types ArchiveType? iat = GetCurrentArchiveType(inputArchive, logger); - ArchiveType? oat = (System.IO.File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip); + ArchiveType? oat = (File.Exists(outputArchive) ? GetCurrentArchiveType(outputArchive, logger) : ArchiveType.Zip); // If we got back null (or the output is not a Zipfile), then it's not an archive, so we we return if (iat == null || (oat == null || oat != ArchiveType.Zip) || inputArchive == outputArchive) @@ -417,7 +417,7 @@ namespace SabreTools.Helper try { - using (IReader reader = ReaderFactory.Open(System.IO.File.OpenRead(inputArchive))) + using (IReader reader = ReaderFactory.Open(File.OpenRead(inputArchive))) { if (iat == ArchiveType.Zip || iat == ArchiveType.SevenZip || iat == ArchiveType.Rar) { @@ -427,7 +427,7 @@ namespace SabreTools.Helper if (reader.Entry != null && reader.Entry.Key.Contains(sourceEntryName)) { // Get if the file should be written out - bool newfile = System.IO.File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0; + bool newfile = File.Exists(outputArchive) && new FileInfo(outputArchive).Length != 0; using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile ? ArchiveFactory.Open(outputArchive, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive @@ -447,10 +447,10 @@ namespace SabreTools.Helper } } - if (System.IO.File.Exists(outputArchive + ".tmp")) + if (File.Exists(outputArchive + ".tmp")) { - System.IO.File.Delete(outputArchive); - System.IO.File.Move(outputArchive + ".tmp", outputArchive); + File.Delete(outputArchive); + File.Move(outputArchive + ".tmp", outputArchive); } success = true; @@ -512,7 +512,7 @@ namespace SabreTools.Helper if (at == ArchiveType.GZip) { // Get the CRC and size from the file - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) { br.BaseStream.Seek(-8, SeekOrigin.End); byte[] headercrc = br.ReadBytes(4); @@ -522,7 +522,7 @@ namespace SabreTools.Helper } } - reader = ReaderFactory.Open(System.IO.File.OpenRead(input)); + reader = ReaderFactory.Open(File.OpenRead(input)); if (at != ArchiveType.Tar) { @@ -594,7 +594,7 @@ namespace SabreTools.Helper byte[] headermd5; // MD5 byte[] headercrc; // CRC byte[] headersz; // Int64 size - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) { header = br.ReadBytes(12); headermd5 = br.ReadBytes(16); @@ -650,7 +650,7 @@ namespace SabreTools.Helper public static bool WriteTorrentGZ(string input, string outdir, bool romba, Logger logger) { // Check that the input file exists - if (!System.IO.File.Exists(input)) + if (!File.Exists(input)) { logger.Warning("File " + input + " does not exist!"); return false; @@ -670,7 +670,7 @@ namespace SabreTools.Helper // If it doesn't exist, create the output file and then write string outfile = Path.Combine(outdir, rom.HashData.SHA1 + ".gz"); using (FileStream inputstream = new FileStream(input, FileMode.Open)) - using (GZipStream output = new GZipStream(System.IO.File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress)) + using (GZipStream output = new GZipStream(File.Open(outfile, FileMode.Create, FileAccess.Write), CompressionMode.Compress)) { inputstream.CopyTo(output); } @@ -678,7 +678,7 @@ namespace SabreTools.Helper // Now that it's ready, inject the header info using (BinaryWriter sw = new BinaryWriter(new MemoryStream())) { - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(outfile))) + using (BinaryReader br = new BinaryReader(File.OpenRead(outfile))) { // Write standard header and TGZ info byte[] data = Constants.TorrentGZHeader @@ -698,7 +698,7 @@ namespace SabreTools.Helper } } - using (BinaryWriter bw = new BinaryWriter(System.IO.File.Open(outfile, FileMode.Create))) + using (BinaryWriter bw = new BinaryWriter(File.Open(outfile, FileMode.Create))) { // Now write the new file over the original sw.BaseStream.Seek(0, SeekOrigin.Begin); @@ -719,12 +719,12 @@ namespace SabreTools.Helper try { - System.IO.File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile))); + File.Move(outfile, Path.Combine(outdir, Path.GetFileName(outfile))); } catch (Exception ex) { logger.Warning(ex.ToString()); - System.IO.File.Delete(outfile); + File.Delete(outfile); } } @@ -754,7 +754,7 @@ namespace SabreTools.Helper try { byte[] magic = new byte[8]; - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) { magic = br.ReadBytes(8); } @@ -851,5 +851,295 @@ namespace SabreTools.Helper bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16); return bytes; } + + #region Hash-to-Dat functions + + // All things in this region are direct ports and do not take advantage of the multiple rom per hash that comes with the new system + + /// + /// Copy a file to an output archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// This uses the new system that is not implemented anywhere yet + public static void WriteToArchive(string input, string output, RomData rom, int machineIndex) + { + string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip"); + + ZipArchive outarchive = null; + try + { + if (!File.Exists(archiveFileName)) + { + outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Create); + } + else + { + outarchive = ZipFile.Open(archiveFileName, ZipArchiveMode.Update); + } + + if (File.Exists(input)) + { + if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(rom.Name) == null) + { + outarchive.CreateEntryFromFile(input, rom.Name, CompressionLevel.Optimal); + } + } + else if (Directory.Exists(input)) + { + foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) + { + if (outarchive.Mode == ZipArchiveMode.Create || outarchive.GetEntry(file) == null) + { + outarchive.CreateEntryFromFile(file, file, CompressionLevel.Optimal); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex); + } + finally + { + outarchive?.Dispose(); + } + } + + /// + /// Copy a file to an output archive using SharpCompress + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// This uses the new system that is not implemented anywhere yet + public static void WriteToManagedArchive(string input, string output, RomData rom, int machineIndex) + { + string archiveFileName = Path.Combine(output, rom.Machines[machineIndex] + ".zip"); + + // Delete an empty file first + if (File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length == 0) + { + File.Delete(archiveFileName); + } + + // Get if the file should be written out + bool newfile = File.Exists(archiveFileName) && new FileInfo(archiveFileName).Length != 0; + + using (SharpCompress.Archive.Zip.ZipArchive archive = (newfile + ? ArchiveFactory.Open(archiveFileName, Options.LookForHeader) as SharpCompress.Archive.Zip.ZipArchive + : ArchiveFactory.Create(ArchiveType.Zip) as SharpCompress.Archive.Zip.ZipArchive)) + { + try + { + if (File.Exists(input)) + { + archive.AddEntry(rom.Name, input); + } + else if (Directory.Exists(input)) + { + archive.AddAllFromDirectory(input, "*", SearchOption.AllDirectories); + } + + archive.SaveTo(archiveFileName + ".tmp", CompressionType.Deflate); + } + catch (Exception) + { + // Don't log archive write errors + } + } + + if (File.Exists(archiveFileName + ".tmp")) + { + File.Delete(archiveFileName); + File.Move(archiveFileName + ".tmp", archiveFileName); + } + } + + /// + /// Generate a list of HashData objects from the header values in an archive + /// + /// Input file to get data from + /// Logger object for file and console output + /// List of HashData objects representing the found data + /// This uses the new system that is not implemented anywhere yet + public static List GetArchiveFileHashes(string input, Logger logger) + { + List hashes = new List(); + string gamename = Path.GetFileNameWithoutExtension(input); + + // First get the archive type + ArchiveType? at = GetCurrentArchiveType(input, logger); + + // If we got back null, then it's not an archive, so we we return + if (at == null) + { + return hashes; + } + + // If we got back GZip, try to get TGZ info first + else if (at == ArchiveType.GZip) + { + HashData possibleTgz = GetTorrentGZFileHash(input, logger); + + // If it was, then add it to the outputs and continue + if (possibleTgz.Size != -1) + { + hashes.Add(possibleTgz); + return hashes; + } + } + + IReader reader = null; + try + { + logger.Log("Found archive of type: " + at); + long size = 0; + byte[] crc = null; + + // If we have a gzip file, get the crc directly + if (at == ArchiveType.GZip) + { + // Get the CRC and size from the file + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) + { + br.BaseStream.Seek(-8, SeekOrigin.End); + crc = br.ReadBytes(4).Reverse().ToArray(); + byte[] headersize = br.ReadBytes(4); + size = BitConverter.ToInt32(headersize.Reverse().ToArray(), 0); + } + } + + reader = ReaderFactory.Open(File.OpenRead(input)); + + if (at != ArchiveType.Tar) + { + while (reader.MoveToNextEntry()) + { + if (reader.Entry != null && !reader.Entry.IsDirectory) + { + logger.Log("Entry found: '" + reader.Entry.Key + "': " + + (size == 0 ? reader.Entry.Size : size) + ", " + + (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc)); + + MachineData tempmachine = new MachineData + { + Name = gamename, + Description = gamename, + }; + RomData temprom = new RomData + { + Type = ItemType.Rom, + Name = reader.Entry.Key, + Machines = new List(), + }; + temprom.Machines.Add(tempmachine); + HashData temphash = new HashData + { + Size = (size == 0 ? reader.Entry.Size : size), + CRC = (crc == null ? BitConverter.GetBytes(reader.Entry.Crc) : crc), + MD5 = null, + SHA1 = null, + Roms = new List(), + }; + temphash.Roms.Add(temprom); + hashes.Add(temphash); + } + } + } + } + catch (Exception ex) + { + logger.Error(ex.ToString()); + } + finally + { + reader?.Dispose(); + } + + return hashes; + } + + /// + /// Retrieve file information for a single torrent GZ file + /// + /// Filename to get information from + /// Logger object for file and console output + /// Populated HashData object if success, empty one on error + /// This uses the new system that is not implemented anywhere yet + public static HashData GetTorrentGZFileHash(string input, Logger logger) + { + string datum = Path.GetFileName(input).ToLowerInvariant(); + string sha1 = Path.GetFileNameWithoutExtension(input).ToLowerInvariant(); + long filesize = new FileInfo(input).Length; + + // Check if the name is the right length + if (!Regex.IsMatch(datum, @"^[0-9a-f]{40}\.gz")) + { + logger.Warning("Non SHA-1 filename found, skipping: '" + datum + "'"); + return new HashData(); + } + + // Check if the file is at least the minimum length + if (filesize < 40 /* bytes */) + { + logger.Warning("Possibly corrupt file '" + input + "' with size " + Style.GetBytesReadable(filesize)); + return new HashData(); + } + + // Get the Romba-specific header data + byte[] header; // Get preamble header for checking + byte[] headermd5; // MD5 + byte[] headercrc; // CRC + byte[] headersz; // Int64 size + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) + { + header = br.ReadBytes(12); + headermd5 = br.ReadBytes(16); + headercrc = br.ReadBytes(4); + headersz = br.ReadBytes(8); + } + + // If the header is not correct, return a blank rom + bool correct = true; + for (int i = 0; i < header.Length; i++) + { + correct &= (header[i] == Constants.TorrentGZHeader[i]); + } + if (!correct) + { + return new HashData(); + } + + // Now convert the size and get the right position + long extractedsize = (long)BitConverter.ToUInt64(headersz.Reverse().ToArray(), 0); + + MachineData tempmachine = new MachineData + { + Name = sha1, + Description = sha1, + }; + RomData temprom = new RomData + { + Type = ItemType.Rom, + Name = sha1, + Machines = new List(), + }; + temprom.Machines.Add(tempmachine); + HashData temphash = new HashData + { + Size = extractedsize, + CRC = headercrc, + MD5 = headermd5, + SHA1 = StringToByteArray(sha1), + Roms = new List(), + }; + temphash.Roms.Add(temprom); + + return temphash; + } + + #endregion } } diff --git a/SabreTools.Helper/Tools/DBTools.cs b/SabreTools.Helper/Tools/DBTools.cs index 13eaafcb..20fb09aa 100644 --- a/SabreTools.Helper/Tools/DBTools.cs +++ b/SabreTools.Helper/Tools/DBTools.cs @@ -17,7 +17,7 @@ namespace SabreTools.Helper public static void EnsureDatabase(string db, string connectionString) { // Make sure the file exists - if (!System.IO.File.Exists(db)) + if (!File.Exists(db)) { SqliteConnection.CreateFile(db); } diff --git a/SabreTools.Helper/Tools/DatTools.cs b/SabreTools.Helper/Tools/DatTools.cs index ec27c174..dfe86cca 100644 --- a/SabreTools.Helper/Tools/DatTools.cs +++ b/SabreTools.Helper/Tools/DatTools.cs @@ -19,7 +19,7 @@ namespace SabreTools.Helper { try { - StreamReader sr = System.IO.File.OpenText(filename); + StreamReader sr = File.OpenText(filename); string first = sr.ReadLine(); sr.Close(); sr.Dispose(); @@ -53,7 +53,7 @@ namespace SabreTools.Helper logger.Log("Attempting to read file: \"" + filename + "\""); // Check if file exists - if (!System.IO.File.Exists(filename)) + if (!File.Exists(filename)) { logger.Warning("File '" + filename + "' could not read from!"); return null; @@ -130,14 +130,14 @@ namespace SabreTools.Helper logger.Log("Attempting to read file: \"" + filename + "\""); // Check if file exists - if (!System.IO.File.Exists(filename)) + if (!File.Exists(filename)) { logger.Warning("File '" + filename + "' could not read from!"); return datdata; } // If it does, open a file reader - StreamReader sr = new StreamReader(System.IO.File.OpenRead(filename)); + StreamReader sr = new StreamReader(File.OpenRead(filename)); bool block = false, superdat = false; string blockname = "", gamename = "", gamedesc = ""; @@ -457,14 +457,14 @@ namespace SabreTools.Helper logger.Log("Attempting to read file: \"" + filename + "\""); // Check if file exists - if (!System.IO.File.Exists(filename)) + if (!File.Exists(filename)) { logger.Warning("File '" + filename + "' could not read from!"); return datdata; } // If it does, open a file reader - StreamReader sr = new StreamReader(System.IO.File.OpenRead(filename)); + StreamReader sr = new StreamReader(File.OpenRead(filename)); string blocktype = ""; while (!sr.EndOfStream) @@ -1627,7 +1627,7 @@ namespace SabreTools.Helper } } } - else if (System.IO.File.Exists(input)) + else if (File.Exists(input)) { try { @@ -1687,7 +1687,7 @@ namespace SabreTools.Helper inputFileName = Path.GetFullPath(inputFileName); } - if (System.IO.File.Exists(inputFileName)) + if (File.Exists(inputFileName)) { logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\""); datdata = Parse(inputFileName, 0, 0, datdata, logger, true, clean, softlist, keepext:(datdata.XSV != null)); @@ -2280,5 +2280,1424 @@ namespace SabreTools.Helper Output.WriteDatfile(userData, outdir, logger); } } + + #region Hash-to-Dat functions + + /// + /// Parse a DAT and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// The DatData object representing found roms to this point + /// Logger object for console and/or file output + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// DatData object representing the read-in data + /// This uses the new system that is not implemented anywhere yet + public static DatData ParseNew(string filename, int sysid, int srcid, DatData datdata, Logger logger, bool keep = false, bool clean = false, bool softlist = false, bool keepext = false) + { + // Check the file extension first as a safeguard + string ext = Path.GetExtension(filename).ToLowerInvariant(); + if (ext != ".txt" && ext != ".dat" && ext != ".xml") + { + return datdata; + } + + // If the output filename isn't set already, get the internal filename + datdata.FileName = (String.IsNullOrEmpty(datdata.FileName) ? (keepext ? Path.GetFileName(filename) : Path.GetFileNameWithoutExtension(filename)) : datdata.FileName); + + // If the output type isn't set already, get the internal output type + datdata.OutputFormat = (datdata.OutputFormat == OutputFormat.None ? GetOutputFormat(filename) : datdata.OutputFormat); + + // Make sure there's a dictionary to read to + if (datdata.Hashes == null) + { + datdata.Hashes = new List(); + } + + // Now parse the correct type of DAT + switch (GetOutputFormat(filename)) + { + case OutputFormat.ClrMamePro: + return ParseCMPNew(filename, sysid, srcid, datdata, logger, keep, clean); + case OutputFormat.RomCenter: + return ParseRCNew(filename, sysid, srcid, datdata, logger, clean); + case OutputFormat.SabreDat: + case OutputFormat.Xml: + return ParseXMLNew(filename, sysid, srcid, datdata, logger, keep, clean, softlist); + default: + return datdata; + } + } + + /// + /// Parse a ClrMamePro DAT and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// The DatData object representing found roms to this point + /// Logger object for console and/or file output + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// DatData object representing the read-in data + /// This uses the new system that is not implemented anywhere yet + public static DatData ParseCMPNew(string filename, int sysid, int srcid, DatData datdata, Logger logger, bool keep, bool clean) + { + // Read the input file, if possible + logger.Log("Attempting to read file: \"" + filename + "\""); + + // Check if file exists + if (!File.Exists(filename)) + { + logger.Warning("File '" + filename + "' could not read from!"); + return datdata; + } + + // If it does, open a file reader + StreamReader sr = new StreamReader(File.OpenRead(filename)); + + bool block = false, superdat = false; + string blockname = "", gamename = "", gamedesc = ""; + while (!sr.EndOfStream) + { + string line = sr.ReadLine(); + + // Comments in CMP DATs start with a # + if (line.Trim().StartsWith("#")) + { + continue; + } + + // If the line is the header or a game + if (Regex.IsMatch(line, Constants.HeaderPatternCMP)) + { + GroupCollection gc = Regex.Match(line, Constants.HeaderPatternCMP).Groups; + + if (gc[1].Value == "clrmamepro" || gc[1].Value == "romvault") + { + blockname = "header"; + } + + block = true; + } + + // If the line is a rom or disk and we're in a block + else if ((line.Trim().StartsWith("rom (") || line.Trim().StartsWith("disk (")) && block) + { + // If we're in cleaning mode, sanitize the game name + gamename = (clean ? Style.CleanGameName(gamename) : gamename); + + MachineData machineData = new MachineData + { + Name = gamename, + Description = gamedesc, + SystemID = sysid, + SourceID = srcid, + }; + RomData romData = new RomData + { + Type = (line.Trim().StartsWith("disk (") ? ItemType.Disk : ItemType.Rom), + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Roms = new List(), + }; + + string[] gc = line.Trim().Split(' '); + + // Loop over all attributes and add them if possible + bool quote = false; + string attrib = "", val = ""; + for (int i = 2; i < gc.Length; i++) + { + //If the item is empty, we automatically skip it because it's a fluke + if (gc[i].Trim() == String.Empty) + { + continue; + } + // Special case for nodump... + else if (gc[i] == "nodump" && attrib != "status" && attrib != "flags") + { + romData.Nodump = true; + } + // Even number of quotes, not in a quote, not in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 0 && !quote && attrib == "") + { + attrib = gc[i].Replace("\"", ""); + } + // Even number of quotes, not in a quote, in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 0 && !quote && attrib != "") + { + switch (attrib.ToLowerInvariant()) + { + case "name": + romData.Name = gc[i].Replace("\"", ""); + break; + case "size": + Int64.TryParse(gc[i].Replace("\"", ""), out hashData.Size); + break; + case "crc": + hashData.CRC = ArchiveTools.StringToByteArray(gc[i].Replace("\"", "")); + break; + case "md5": + hashData.MD5 = ArchiveTools.StringToByteArray(gc[i].Replace("\"", "")); + break; + case "sha1": + hashData.SHA1 = ArchiveTools.StringToByteArray(gc[i].Replace("\"", "")); + break; + case "flags": + if (gc[i].Replace("\"", "").ToLowerInvariant() == "nodump") + { + romData.Nodump = true; + } + break; + } + + attrib = ""; + } + // Even number of quotes, in a quote, not in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 0 && quote && attrib == "") + { + // Attributes can't have quoted names + } + // Even number of quotes, in a quote, in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 0 && quote && attrib != "") + { + val += " " + gc[i]; + } + // Odd number of quotes, not in a quote, not in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 1 && !quote && attrib == "") + { + // Attributes can't have quoted names + } + // Odd number of quotes, not in a quote, in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 1 && !quote && attrib != "") + { + val = gc[i].Replace("\"", ""); + quote = true; + } + // Odd number of quotes, in a quote, not in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 1 && quote && attrib == "") + { + quote = false; + } + // Odd number of quotes, in a quote, in attribute + else if (Regex.Matches(gc[i], "\"").Count % 2 == 1 && quote && attrib != "") + { + val += " " + gc[i].Replace("\"", ""); + switch (attrib.ToLowerInvariant()) + { + case "name": + romData.Name = val; + break; + case "size": + Int64.TryParse(val, out hashData.Size); + break; + case "crc": + hashData.CRC = ArchiveTools.StringToByteArray(val); + break; + case "md5": + hashData.MD5 = ArchiveTools.StringToByteArray(val); + break; + case "sha1": + hashData.SHA1 = ArchiveTools.StringToByteArray(val); + break; + case "flags": + if (val.ToLowerInvariant() == "nodump") + { + romData.Nodump = true; + } + break; + } + + quote = false; + attrib = ""; + val = ""; + } + } + + // Sanitize the hashes from null, hex sizes, and "true blank" strings + hashData.CRC = RomTools.CleanHashData(hashData.CRC, Constants.CRCBytesLength); + hashData.MD5 = RomTools.CleanHashData(hashData.MD5, Constants.MD5BytesLength); + hashData.SHA1 = RomTools.CleanHashData(hashData.SHA1, Constants.SHA1BytesLength); + + // If we have a rom and it's missing size AND the hashes match a 0-byte file, fill in the rest of the info + if (romData.Type == ItemType.Rom + && (hashData.Size == 0 || hashData.Size == -1) + && ((hashData.CRC == Constants.CRCZeroBytes || hashData.CRC == null) + || hashData.MD5 == Constants.MD5ZeroBytes + || hashData.SHA1 == Constants.SHA1ZeroBytes)) + { + hashData.Size = Constants.SizeZero; + hashData.CRC = Constants.CRCZeroBytes; + hashData.MD5 = Constants.MD5ZeroBytes; + hashData.SHA1 = Constants.SHA1ZeroBytes; + } + // If the file has no size and it's not the above case, skip and log + else if (romData.Type == ItemType.Rom && (hashData.Size == 0 || hashData.Size == -1)) + { + logger.Warning("Incomplete entry for \"" + romData.Name + "\" will be output as nodump"); + romData.Nodump = true; + } + + // If we have a disk, make sure that the value for size is -1 + if (romData.Type == ItemType.Disk) + { + hashData.Size = -1; + } + + // Now add the hash to the Dat + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + // If the line is anything but a rom or disk and we're in a block + else if (Regex.IsMatch(line, Constants.ItemPatternCMP) && block) + { + GroupCollection gc = Regex.Match(line, Constants.ItemPatternCMP).Groups; + + if (gc[1].Value == "name" && blockname != "header") + { + gamename = gc[2].Value.Replace("\"", ""); + } + else if (gc[1].Value == "description" && blockname != "header") + { + gamedesc = gc[2].Value.Replace("\"", ""); + } + else + { + string itemval = gc[2].Value.Replace("\"", ""); + switch (gc[1].Value) + { + case "name": + datdata.Name = (String.IsNullOrEmpty(datdata.Name) ? itemval : datdata.Name); + superdat = superdat || itemval.Contains(" - SuperDAT"); + if (keep && superdat) + { + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? "SuperDAT" : datdata.Type); + } + break; + case "description": + datdata.Description = (String.IsNullOrEmpty(datdata.Description) ? itemval : datdata.Description); + break; + case "rootdir": + datdata.RootDir = (String.IsNullOrEmpty(datdata.RootDir) ? itemval : datdata.RootDir); + break; + case "category": + datdata.Category = (String.IsNullOrEmpty(datdata.Category) ? itemval : datdata.Category); + break; + case "version": + datdata.Version = (String.IsNullOrEmpty(datdata.Version) ? itemval : datdata.Version); + break; + case "date": + datdata.Date = (String.IsNullOrEmpty(datdata.Date) ? itemval : datdata.Date); + break; + case "author": + datdata.Author = (String.IsNullOrEmpty(datdata.Author) ? itemval : datdata.Author); + break; + case "email": + datdata.Email = (String.IsNullOrEmpty(datdata.Email) ? itemval : datdata.Email); + break; + case "homepage": + datdata.Homepage = (String.IsNullOrEmpty(datdata.Homepage) ? itemval : datdata.Homepage); + break; + case "url": + datdata.Url = (String.IsNullOrEmpty(datdata.Url) ? itemval : datdata.Url); + break; + case "comment": + datdata.Comment = (String.IsNullOrEmpty(datdata.Comment) ? itemval : datdata.Comment); + break; + case "header": + datdata.Header = (String.IsNullOrEmpty(datdata.Header) ? itemval : datdata.Header); + break; + case "type": + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? itemval : datdata.Type); + superdat = superdat || itemval.Contains("SuperDAT"); + break; + case "forcemerging": + switch (itemval) + { + case "none": + datdata.ForceMerging = ForceMerging.None; + break; + case "split": + datdata.ForceMerging = ForceMerging.Split; + break; + case "full": + datdata.ForceMerging = ForceMerging.Full; + break; + } + break; + case "forcezipping": + datdata.ForcePacking = (itemval == "yes" ? ForcePacking.Zip : ForcePacking.Unzip); + break; + } + } + } + + // If we find an end bracket that's not associated with anything else, the block is done + else if (Regex.IsMatch(line, Constants.EndPatternCMP) && block) + { + block = false; + blockname = ""; + gamename = ""; + } + } + + sr.Close(); + sr.Dispose(); + + return datdata; + } + + /// + /// Parse a RomCenter DAT and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// The DatData object representing found roms to this point + /// Logger object for console and/or file output + /// True if game names are sanitized, false otherwise (default) + /// DatData object representing the read-in data + /// This uses the new system that is not implemented anywhere yet + public static DatData ParseRCNew(string filename, int sysid, int srcid, DatData datdata, Logger logger, bool clean) + { + // Read the input file, if possible + logger.Log("Attempting to read file: \"" + filename + "\""); + + // Check if file exists + if (!File.Exists(filename)) + { + logger.Warning("File '" + filename + "' could not read from!"); + return datdata; + } + + // If it does, open a file reader + StreamReader sr = new StreamReader(File.OpenRead(filename)); + + string blocktype = ""; + while (!sr.EndOfStream) + { + string line = sr.ReadLine(); + + // If the line is the start of the credits section + if (line.ToLowerInvariant().Contains("[credits]")) + { + blocktype = "credits"; + } + // If the line is the start of the dat section + else if (line.ToLowerInvariant().Contains("[dat]")) + { + blocktype = "dat"; + } + // If the line is the start of the emulator section + else if (line.ToLowerInvariant().Contains("[emulator]")) + { + blocktype = "emulator"; + } + // If the line is the start of the game section + else if (line.ToLowerInvariant().Contains("[games]")) + { + blocktype = "games"; + } + // Otherwise, it's not a section and it's data, so get out all data + else + { + // If we have an author + if (line.StartsWith("author=")) + { + datdata.Author = (String.IsNullOrEmpty(datdata.Author) ? line.Split('=')[1] : datdata.Author); + } + // If we have one of the three version tags + else if (line.StartsWith("version=")) + { + switch (blocktype) + { + case "credits": + datdata.Version = (String.IsNullOrEmpty(datdata.Version) ? line.Split('=')[1] : datdata.Version); + break; + case "emulator": + datdata.Description = (String.IsNullOrEmpty(datdata.Description) ? line.Split('=')[1] : datdata.Description); + break; + } + } + // If we have a comment + else if (line.StartsWith("comment=")) + { + datdata.Comment = (String.IsNullOrEmpty(datdata.Comment) ? line.Split('=')[1] : datdata.Comment); + } + // If we have the split flag + else if (line.StartsWith("split=")) + { + int split = 0; + if (Int32.TryParse(line.Split('=')[1], out split)) + { + if (split == 1) + { + datdata.ForceMerging = ForceMerging.Split; + } + } + } + // If we have the merge tag + else if (line.StartsWith("merge=")) + { + int merge = 0; + if (Int32.TryParse(line.Split('=')[1], out merge)) + { + if (merge == 1) + { + datdata.ForceMerging = ForceMerging.Full; + } + } + } + // If we have the refname tag + else if (line.StartsWith("refname=")) + { + datdata.Name = (String.IsNullOrEmpty(datdata.Name) ? line.Split('=')[1] : datdata.Name); + } + // If we have a rom + else if (line.StartsWith("¬")) + { + /* + The rominfo order is as follows: + 1 - parent name + 2 - parent description + 3 - game name + 4 - game description + 5 - rom name + 6 - rom crc + 7 - rom size + 8 - romof name + 9 - merge name + */ + string[] rominfo = line.Split('¬'); + + // If we're in cleaning mode, sanitize the game name + rominfo[3] = (clean ? Style.CleanGameName(rominfo[3]) : rominfo[3]); + + MachineData machineData = new MachineData + { + Name = rominfo[3], + Description = rominfo[4], + CloneOf = rominfo[1], + RomOf = rominfo[1], + SystemID = sysid, + SourceID = srcid, + }; + RomData romData = new RomData + { + Name = rominfo[5], + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Size = Int64.Parse(rominfo[7]), + CRC = ArchiveTools.StringToByteArray(rominfo[6]), + }; + + // Sanitize the hashes from null, hex sizes, and "true blank" strings + hashData.CRC = RomTools.CleanHashData(hashData.CRC, Constants.CRCBytesLength); + hashData.MD5 = RomTools.CleanHashData(hashData.MD5, Constants.MD5BytesLength); + hashData.SHA1 = RomTools.CleanHashData(hashData.SHA1, Constants.SHA1BytesLength); + + // If we have a rom and it's missing size AND the hashes match a 0-byte file, fill in the rest of the info + if (romData.Type == ItemType.Rom + && (hashData.Size == 0 || hashData.Size == -1) + && ((hashData.CRC == Constants.CRCZeroBytes || hashData.CRC == null) + || hashData.MD5 == Constants.MD5ZeroBytes + || hashData.SHA1 == Constants.SHA1ZeroBytes)) + { + hashData.Size = Constants.SizeZero; + hashData.CRC = Constants.CRCZeroBytes; + hashData.MD5 = Constants.MD5ZeroBytes; + hashData.SHA1 = Constants.SHA1ZeroBytes; + } + // If the file has no size and it's not the above case, skip and log + else if (romData.Type == ItemType.Rom && (hashData.Size == 0 || hashData.Size == -1)) + { + logger.Warning("Incomplete entry for \"" + romData.Name + "\" will be output as nodump"); + romData.Nodump = true; + } + + // If we have a disk, make sure that the value for size is -1 + if (romData.Type == ItemType.Disk) + { + hashData.Size = -1; + } + + // Now add the hash to the Dat + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + } + } + + sr.Close(); + sr.Dispose(); + + return datdata; + } + + /// + /// Parse an XML DAT (Logiqx, SabreDAT, or SL) and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// The DatData object representing found roms to this point + /// Logger object for console and/or file output + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if SL XML names should be kept, false otherwise (default) + /// DatData object representing the read-in data + /// This uses the new system that is not implemented anywhere yet + public static DatData ParseXMLNew(string filename, int sysid, int srcid, DatData datdata, Logger logger, bool keep, bool clean, bool softlist) + { + // Prepare all internal variables + XmlReader subreader, headreader, flagreader; + bool superdat = false, nodump = false, empty = true; + string crc = "", md5 = "", sha1 = "", date = ""; + long size = -1; + List parent = new List(); + + XmlTextReader xtr = GetXmlTextReader(filename, logger); + if (xtr != null) + { + xtr.MoveToContent(); + while (!xtr.EOF) + { + // If we're ending a folder or game, take care of possibly empty games and removing from the parent + if (xtr.NodeType == XmlNodeType.EndElement && (xtr.Name == "directory" || xtr.Name == "dir")) + { + // If we didn't find any items in the folder, make sure to add the blank rom + if (empty) + { + string tempgame = String.Join("\\", parent); + + // If we're in cleaning mode, sanitize the game name + tempgame = (clean ? Style.CleanGameName(tempgame) : tempgame); + + MachineData machineData = new MachineData + { + Name = tempgame, + Description = tempgame, + }; + RomData romData = new RomData + { + Type = ItemType.Rom, + Name = "null", + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Size = -1, + CRC = null, + MD5 = null, + SHA1 = null, + Roms = new List(), + }; + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + + // Regardless, end the current folder + int parentcount = parent.Count; + if (parentcount == 0) + { + logger.Log("Empty parent: " + String.Join("\\", parent)); + empty = true; + } + + // If we have an end folder element, remove one item from the parent, if possible + if (parentcount > 0) + { + parent.RemoveAt(parent.Count - 1); + if (keep && parentcount > 1) + { + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? "SuperDAT" : datdata.Type); + superdat = true; + } + } + } + + // We only want elements + if (xtr.NodeType != XmlNodeType.Element) + { + xtr.Read(); + continue; + } + + switch (xtr.Name) + { + // New software lists have this behavior + case "softwarelist": + if (xtr.GetAttribute("name") != null) + { + datdata.Name = (String.IsNullOrEmpty(datdata.Name) ? xtr.GetAttribute("name") : datdata.Name); + } + if (xtr.GetAttribute("description") != null) + { + datdata.Description = (String.IsNullOrEmpty(datdata.Description) ? xtr.GetAttribute("description") : datdata.Description); + } + xtr.Read(); + break; + // Handle M1 DATs since they're 99% the same as a SL DAT + case "m1": + datdata.Name = (String.IsNullOrEmpty(datdata.Name) ? "M1" : datdata.Name); + datdata.Description = (String.IsNullOrEmpty(datdata.Description) ? "M1" : datdata.Description); + if (xtr.GetAttribute("version") != null) + { + datdata.Version = (String.IsNullOrEmpty(datdata.Version) ? xtr.GetAttribute("version") : datdata.Version); + } + break; + case "header": + // We want to process the entire subtree of the header + headreader = xtr.ReadSubtree(); + + if (headreader != null) + { + while (!headreader.EOF) + { + // We only want elements + if (headreader.NodeType != XmlNodeType.Element || headreader.Name == "header") + { + headreader.Read(); + continue; + } + + // Get all header items (ONLY OVERWRITE IF THERE'S NO DATA) + string content = ""; + switch (headreader.Name) + { + case "name": + content = headreader.ReadElementContentAsString(); ; + datdata.Name = (String.IsNullOrEmpty(datdata.Name) ? content : datdata.Name); + superdat = superdat || content.Contains(" - SuperDAT"); + if (keep && superdat) + { + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? "SuperDAT" : datdata.Type); + } + break; + case "description": + content = headreader.ReadElementContentAsString(); + datdata.Description = (String.IsNullOrEmpty(datdata.Description) ? content : datdata.Description); + break; + case "rootdir": + content = headreader.ReadElementContentAsString(); + datdata.RootDir = (String.IsNullOrEmpty(datdata.RootDir) ? content : datdata.RootDir); + break; + case "category": + content = headreader.ReadElementContentAsString(); + datdata.Category = (String.IsNullOrEmpty(datdata.Category) ? content : datdata.Category); + break; + case "version": + content = headreader.ReadElementContentAsString(); + datdata.Version = (String.IsNullOrEmpty(datdata.Version) ? content : datdata.Version); + break; + case "date": + content = headreader.ReadElementContentAsString(); + datdata.Date = (String.IsNullOrEmpty(datdata.Date) ? content : datdata.Date); + break; + case "author": + content = headreader.ReadElementContentAsString(); + datdata.Author = (String.IsNullOrEmpty(datdata.Author) ? content : datdata.Author); + + // Special cases for SabreDAT + datdata.Email = (String.IsNullOrEmpty(datdata.Email) && !String.IsNullOrEmpty(headreader.GetAttribute("email")) ? + headreader.GetAttribute("email") : datdata.Email); + datdata.Homepage = (String.IsNullOrEmpty(datdata.Homepage) && !String.IsNullOrEmpty(headreader.GetAttribute("homepage")) ? + headreader.GetAttribute("homepage") : datdata.Email); + datdata.Url = (String.IsNullOrEmpty(datdata.Url) && !String.IsNullOrEmpty(headreader.GetAttribute("url")) ? + headreader.GetAttribute("url") : datdata.Email); + break; + case "email": + content = headreader.ReadElementContentAsString(); + datdata.Email = (String.IsNullOrEmpty(datdata.Email) ? content : datdata.Email); + break; + case "homepage": + content = headreader.ReadElementContentAsString(); + datdata.Homepage = (String.IsNullOrEmpty(datdata.Homepage) ? content : datdata.Homepage); + break; + case "url": + content = headreader.ReadElementContentAsString(); + datdata.Url = (String.IsNullOrEmpty(datdata.Url) ? content : datdata.Url); + break; + case "comment": + content = headreader.ReadElementContentAsString(); + datdata.Comment = (String.IsNullOrEmpty(datdata.Comment) ? content : datdata.Comment); + break; + case "type": + content = headreader.ReadElementContentAsString(); + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? content : datdata.Type); + superdat = superdat || content.Contains("SuperDAT"); + break; + case "clrmamepro": + if (headreader.GetAttribute("header") != null) + { + datdata.Header = (String.IsNullOrEmpty(datdata.Header) ? headreader.GetAttribute("header") : datdata.Header); + } + if (headreader.GetAttribute("forcemerging") != null) + { + switch (headreader.GetAttribute("forcemerging")) + { + case "split": + datdata.ForceMerging = ForceMerging.Split; + break; + case "none": + datdata.ForceMerging = ForceMerging.None; + break; + case "full": + datdata.ForceMerging = ForceMerging.Full; + break; + } + } + if (headreader.GetAttribute("forcenodump") != null) + { + switch (headreader.GetAttribute("forcenodump")) + { + case "obsolete": + datdata.ForceNodump = ForceNodump.Obsolete; + break; + case "required": + datdata.ForceNodump = ForceNodump.Required; + break; + case "ignore": + datdata.ForceNodump = ForceNodump.Ignore; + break; + } + } + if (headreader.GetAttribute("forcepacking") != null) + { + switch (headreader.GetAttribute("forcepacking")) + { + case "zip": + datdata.ForcePacking = ForcePacking.Zip; + break; + case "unzip": + datdata.ForcePacking = ForcePacking.Unzip; + break; + } + } + headreader.Read(); + break; + case "flags": + flagreader = xtr.ReadSubtree(); + if (flagreader != null) + { + while (!flagreader.EOF) + { + // We only want elements + if (flagreader.NodeType != XmlNodeType.Element || flagreader.Name == "flags") + { + flagreader.Read(); + continue; + } + + switch (flagreader.Name) + { + case "flag": + if (flagreader.GetAttribute("name") != null && flagreader.GetAttribute("value") != null) + { + content = flagreader.GetAttribute("value"); + switch (flagreader.GetAttribute("name")) + { + case "type": + datdata.Type = (String.IsNullOrEmpty(datdata.Type) ? content : datdata.Type); + superdat = superdat || content.Contains("SuperDAT"); + break; + case "forcemerging": + switch (content) + { + case "split": + datdata.ForceMerging = ForceMerging.Split; + break; + case "none": + datdata.ForceMerging = ForceMerging.None; + break; + case "full": + datdata.ForceMerging = ForceMerging.Full; + break; + } + break; + case "forcenodump": + switch (content) + { + case "obsolete": + datdata.ForceNodump = ForceNodump.Obsolete; + break; + case "required": + datdata.ForceNodump = ForceNodump.Required; + break; + case "ignore": + datdata.ForceNodump = ForceNodump.Ignore; + break; + } + break; + case "forcepacking": + switch (content) + { + case "zip": + datdata.ForcePacking = ForcePacking.Zip; + break; + case "unzip": + datdata.ForcePacking = ForcePacking.Unzip; + break; + } + break; + } + } + flagreader.Read(); + break; + default: + flagreader.Read(); + break; + } + } + } + headreader.Skip(); + break; + default: + headreader.Read(); + break; + } + } + } + + // Skip the header node now that we've processed it + xtr.Skip(); + break; + case "machine": + case "game": + case "software": + string temptype = xtr.Name; + string tempname = "", gamedesc = ""; + + // We want to process the entire subtree of the game + subreader = xtr.ReadSubtree(); + + // Safeguard for interesting case of "software" without anything except roms + bool software = false; + + // If we have a subtree, add what is possible + if (subreader != null) + { + if (!softlist && temptype == "software" && subreader.ReadToFollowing("description")) + { + tempname = subreader.ReadElementContentAsString(); + tempname = tempname.Replace('/', '_').Replace("\"", "''"); + software = true; + } + else + { + // There are rare cases where a malformed XML will not have the required attributes. We can only skip them. + if (xtr.AttributeCount == 0) + { + logger.Error("No attributes were found"); + xtr.Skip(); + continue; + } + tempname = xtr.GetAttribute("name"); + } + + if (superdat && !keep) + { + string tempout = Regex.Match(tempname, @".*?\\(.*)").Groups[1].Value; + if (tempout != "") + { + tempname = tempout; + } + } + // Get the name of the game from the parent + else if (superdat && keep && parent.Count > 0) + { + tempname = String.Join("\\", parent) + "\\" + tempname; + } + + while (software || subreader.Read()) + { + software = false; + + // We only want elements + if (subreader.NodeType != XmlNodeType.Element) + { + continue; + } + + // Get the roms from the machine + switch (subreader.Name) + { + case "description": + gamedesc = subreader.ReadElementContentAsString(); + break; + case "rom": + case "disk": + empty = false; + + // If the rom is nodump, flag it + nodump = false; + if (subreader.GetAttribute("flags") == "nodump" || subreader.GetAttribute("status") == "nodump") + { + logger.Log("Nodump detected: " + + (subreader.GetAttribute("name") != null && subreader.GetAttribute("name") != "" ? "\"" + xtr.GetAttribute("name") + "\"" : "ROM NAME NOT FOUND")); + nodump = true; + } + + // If the rom has a Date attached, read it in and then sanitize it + date = ""; + if (subreader.GetAttribute("date") != null) + { + date = DateTime.Parse(subreader.GetAttribute("date")).ToString(); + } + + // Take care of hex-sized files + size = -1; + if (subreader.GetAttribute("size") != null && subreader.GetAttribute("size").Contains("0x")) + { + size = Convert.ToInt64(subreader.GetAttribute("size"), 16); + } + else if (subreader.GetAttribute("size") != null) + { + Int64.TryParse(subreader.GetAttribute("size"), out size); + } + + // If the rom is continue or ignore, add the size to the previous rom + if (subreader.GetAttribute("loadflag") == "continue" || subreader.GetAttribute("loadflag") == "ignore") + { + int index = datdata.Hashes.Count() - 1; + HashData lasthash = datdata.Hashes[index]; + lasthash.Size += size; + datdata.Hashes.RemoveAt(index); + datdata.Hashes.Add(lasthash); + continue; + } + + // Sanitize the hashes from null, hex sizes, and "true blank" strings + crc = RomTools.CleanHashData(subreader.GetAttribute("crc"), Constants.CRCLength); + md5 = RomTools.CleanHashData(subreader.GetAttribute("md5"), Constants.MD5Length); + sha1 = RomTools.CleanHashData(subreader.GetAttribute("sha1"), Constants.SHA1Length); + + // If we have a rom and it's missing size AND the hashes match a 0-byte file, fill in the rest of the info + if (subreader.Name == "rom" && (size == 0 || size == -1) && + ((crc == Constants.CRCZero || crc == "") || md5 == Constants.MD5Zero || sha1 == Constants.SHA1Zero)) + { + size = Constants.SizeZero; + crc = Constants.CRCZero; + md5 = Constants.MD5Zero; + sha1 = Constants.SHA1Zero; + } + // If the file has no size and it's not the above case, skip and log + else if (subreader.Name == "rom" && (size == 0 || size == -1)) + { + logger.Warning("Incomplete entry for \"" + subreader.GetAttribute("name") + "\" will be output as nodump"); + nodump = true; + } + + // If we're in clean mode, sanitize the game name + if (clean) + { + tempname = Style.CleanGameName(tempname.Split(Path.DirectorySeparatorChar)); + } + + // Only add the rom if there's useful information in it + if (!(crc == "" && md5 == "" && sha1 == "") || nodump) + { + // If we got to this point and it's a disk, log it because some tools don't like disks + if (subreader.Name == "disk") + { + logger.Log("Disk found: \"" + subreader.GetAttribute("name") + "\""); + } + + // Get the new values to add + MachineData machineData = new MachineData + { + Name = tempname, + Description = gamedesc, + SystemID = sysid, + System = filename, + SourceID = srcid, + }; + RomData romData = new RomData + { + Name = subreader.GetAttribute("name"), + Type = (subreader.Name.ToLowerInvariant() == "disk" ? ItemType.Disk : ItemType.Rom), + Nodump = nodump, + Date = date, + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Size = size, + CRC = RomTools.CleanHashData(ArchiveTools.StringToByteArray(crc), Constants.CRCBytesLength), + MD5 = RomTools.CleanHashData(ArchiveTools.StringToByteArray(md5), Constants.MD5BytesLength), + SHA1 = RomTools.CleanHashData(ArchiveTools.StringToByteArray(sha1), Constants.SHA1BytesLength), + Roms = new List(), + }; + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + // Otherwise, log that it wasn't added + else + { + logger.Log("Rom was not added: '" + xtr.GetAttribute("name") + "'"); + } + break; + } + } + } + + // If we didn't find any items in the folder, make sure to add the blank rom + if (empty) + { + tempname = (parent.Count > 0 ? String.Join("\\", parent) + Path.DirectorySeparatorChar : "") + tempname; + + // If we're in cleaning mode, sanitize the game name + tempname = (clean ? Style.CleanGameName(tempname.Split(Path.DirectorySeparatorChar)) : tempname); + + MachineData machineData = new MachineData + { + Name = tempname, + Description = tempname, + }; + RomData romData = new RomData + { + Type = ItemType.Rom, + Name = "null", + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Size = -1, + CRC = null, + MD5 = null, + SHA1 = null, + Roms = new List(), + }; + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + + // Regardless, end the current folder + if (parent.Count == 0) + { + empty = true; + } + xtr.Skip(); + break; + case "dir": + case "directory": + // Set SuperDAT flag for all SabreDAT inputs, regardless of depth + superdat = true; + if (keep) + { + datdata.Type = (datdata.Type == "" ? "SuperDAT" : datdata.Type); + } + + string foldername = (xtr.GetAttribute("name") == null ? "" : xtr.GetAttribute("name")); + if (foldername != "") + { + parent.Add(foldername); + } + + xtr.Read(); + break; + case "file": + empty = false; + + // If the rom is nodump, flag it + nodump = false; + flagreader = xtr.ReadSubtree(); + if (flagreader != null) + { + while (!flagreader.EOF) + { + // We only want elements + if (flagreader.NodeType != XmlNodeType.Element || flagreader.Name == "flags") + { + flagreader.Read(); + continue; + } + + switch (flagreader.Name) + { + case "flag": + case "status": + if (flagreader.GetAttribute("name") != null && flagreader.GetAttribute("value") != null) + { + string content = flagreader.GetAttribute("value"); + switch (flagreader.GetAttribute("name")) + { + case "nodump": + logger.Log("Nodump detected: " + (xtr.GetAttribute("name") != null && xtr.GetAttribute("name") != "" ? + "\"" + xtr.GetAttribute("name") + "\"" : "ROM NAME NOT FOUND")); + nodump = true; + break; + } + } + break; + } + + flagreader.Read(); + } + } + + // If the rom has a Date attached, read it in and then sanitize it + date = ""; + if (xtr.GetAttribute("date") != null) + { + date = DateTime.Parse(xtr.GetAttribute("date")).ToString(); + } + + // Take care of hex-sized files + size = -1; + if (xtr.GetAttribute("size") != null && xtr.GetAttribute("size").Contains("0x")) + { + size = Convert.ToInt64(xtr.GetAttribute("size"), 16); + } + else if (xtr.GetAttribute("size") != null) + { + Int64.TryParse(xtr.GetAttribute("size"), out size); + } + + // If the rom is continue or ignore, add the size to the previous rom + if (xtr.GetAttribute("loadflag") == "continue" || xtr.GetAttribute("loadflag") == "ignore") + { + int index = datdata.Hashes.Count() - 1; + HashData lasthash = datdata.Hashes[index]; + lasthash.Size += size; + datdata.Hashes.RemoveAt(index); + datdata.Hashes.Add(lasthash); + continue; + } + + // Sanitize the hashes from null, hex sizes, and "true blank" strings + crc = RomTools.CleanHashData(xtr.GetAttribute("crc"), Constants.CRCLength); + md5 = RomTools.CleanHashData(xtr.GetAttribute("md5"), Constants.MD5Length); + sha1 = RomTools.CleanHashData(xtr.GetAttribute("sha1"), Constants.SHA1Length); + + // If we have a rom and it's missing size AND the hashes match a 0-byte file, fill in the rest of the info + if (xtr.GetAttribute("type") == "rom" && (size == 0 || size == -1) && ((crc == Constants.CRCZero || crc == "") || md5 == Constants.MD5Zero || sha1 == Constants.SHA1Zero)) + { + size = Constants.SizeZero; + crc = Constants.CRCZero; + md5 = Constants.MD5Zero; + sha1 = Constants.SHA1Zero; + } + // If the file has no size and it's not the above case, skip and log + else if (xtr.GetAttribute("type") == "rom" && (size == 0 || size == -1)) + { + logger.Warning("Incomplete entry for \"" + xtr.GetAttribute("name") + "\" will be output as nodump"); + nodump = true; + } + + // Get the name of the game from the parent + tempname = String.Join("\\", parent); + + // If we aren't keeping names, trim out the path + if (!keep || !superdat) + { + string tempout = Regex.Match(tempname, @".*?\\(.*)").Groups[1].Value; + if (tempout != "") + { + tempname = tempout; + } + } + + // If we're in cleaning mode, sanitize the game name + tempname = (clean ? Style.CleanGameName(tempname) : tempname); + + // Only add the rom if there's useful information in it + if (!(crc == "" && md5 == "" && sha1 == "") || nodump) + { + // If we got to this point and it's a disk, log it because some tools don't like disks + if (xtr.GetAttribute("type") == "disk") + { + logger.Log("Disk found: \"" + xtr.GetAttribute("name") + "\""); + } + + // Get the new values to add + MachineData machineData = new MachineData + { + Name = tempname, + SystemID = sysid, + System = filename, + SourceID = srcid, + }; + RomData romData = new RomData + { + Name = xtr.GetAttribute("name"), + Type = (xtr.GetAttribute("type").ToLowerInvariant() == "disk" ? ItemType.Disk : ItemType.Rom), + Nodump = nodump, + Date = date, + Machines = new List(), + }; + romData.Machines.Add(machineData); + HashData hashData = new HashData + { + Size = size, + CRC = RomTools.CleanHashData(ArchiveTools.StringToByteArray(crc), Constants.CRCBytesLength), + MD5 = RomTools.CleanHashData(ArchiveTools.StringToByteArray(md5), Constants.MD5BytesLength), + SHA1 = RomTools.CleanHashData(ArchiveTools.StringToByteArray(sha1), Constants.SHA1BytesLength), + Roms = new List(), + }; + hashData.Roms.Add(romData); + datdata.Hashes.Add(hashData); + + // Add statistical data + datdata.RomCount += (romData.Type == ItemType.Rom ? 1 : 0); + datdata.DiskCount += (romData.Type == ItemType.Disk ? 1 : 0); + datdata.TotalSize += (romData.Nodump ? 0 : hashData.Size); + datdata.CRCCount += (hashData.CRC == null ? 0 : 1); + datdata.MD5Count += (hashData.MD5 == null ? 0 : 1); + datdata.SHA1Count += (hashData.SHA1 == null ? 0 : 1); + datdata.NodumpCount += (romData.Nodump ? 1 : 0); + } + xtr.Read(); + break; + default: + xtr.Read(); + break; + } + } + + xtr.Close(); + xtr.Dispose(); + } + + return datdata; + } + + /// + /// Take an arbitrarily ordered List and return a Dictionary sorted by Game + /// + /// Input unsorted list + /// True if roms should be deduped, false otherwise + /// True if games should only be compared on game and file name, false if system and source are counted + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + /// SortedDictionary bucketed by game name + /// This uses the new system that is not implemented anywhere yet + public static SortedDictionary> BucketByGame(List list, bool mergeroms, bool norename, Logger logger, bool output = true) + { + Dictionary> dict = new Dictionary>(); + dict.Add("key", list); + return BucketByGame(dict, mergeroms, norename, logger, output); + } + + /// + /// Take an arbitrarily bucketed Dictionary and return one sorted by Game + /// + /// Input unsorted dictionary + /// True if roms should be deduped, false otherwise + /// True if games should only be compared on game and file name, false if system and source are counted + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + /// SortedDictionary bucketed by game name + /// This uses the new system that is not implemented anywhere yet + public static SortedDictionary> BucketByGame(IDictionary> dict, bool mergeroms, bool norename, Logger logger, bool output = true) + { + SortedDictionary> sortable = new SortedDictionary>(); + long count = 0; + + // If we have a null dict or an empty one, output a new dictionary + if (dict == null || dict.Count == 0) + { + return sortable; + } + + // Process each all of the roms + foreach (string key in dict.Keys) + { + List hashes = dict[key]; + if (mergeroms) + { + hashes = RomTools.Merge(hashes, logger); + } + + foreach (HashData hash in hashes) + { + count++; + string newkey = (norename ? "" + : hash.Roms[0].Machine.SystemID.ToString().PadLeft(10, '0') + + "-" + + hash.Roms[0].Machine.SourceID.ToString().PadLeft(10, '0') + "-") + + (String.IsNullOrEmpty(hash.Roms[0].Machine.Name) + ? "" + : hash.Roms[0].Machine.Name.ToLowerInvariant()); + if (sortable.ContainsKey(newkey)) + { + sortable[newkey].Add(hash); + } + else + { + List temp = new List(); + temp.Add(hash); + sortable.Add(newkey, temp); + } + } + } + + // Output the count if told to + if (output) + { + logger.User("A total of " + count + " file hashes will be written out to file"); + } + + return sortable; + } + + #endregion } } diff --git a/SabreTools.Helper/Tools/Output.cs b/SabreTools.Helper/Tools/Output.cs index 67cba923..42098b41 100644 --- a/SabreTools.Helper/Tools/Output.cs +++ b/SabreTools.Helper/Tools/Output.cs @@ -85,7 +85,7 @@ namespace SabreTools.Helper try { - FileStream fs = System.IO.File.Create(outfile); + FileStream fs = File.Create(outfile); StreamWriter sw = new StreamWriter(fs, Encoding.UTF8); // Write out the header @@ -619,7 +619,7 @@ namespace SabreTools.Helper { try { - System.IO.File.Delete(file); + File.Delete(file); } catch { } } @@ -643,14 +643,14 @@ namespace SabreTools.Helper public static void RemoveBytesFromFile(string input, string output, long bytesToRemoveFromHead, long bytesToRemoveFromTail) { // If any of the inputs are invalid, skip - if (!System.IO.File.Exists(input) || new FileInfo(input).Length <= (bytesToRemoveFromHead + bytesToRemoveFromTail)) + if (!File.Exists(input) || new FileInfo(input).Length <= (bytesToRemoveFromHead + bytesToRemoveFromTail)) { return; } // Read the input file and write to the fail - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) - using (BinaryWriter bw = new BinaryWriter(System.IO.File.OpenWrite(output))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) + using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output))) { int bufferSize = 1024; long adjustedLength = br.BaseStream.Length - bytesToRemoveFromTail; @@ -708,13 +708,13 @@ namespace SabreTools.Helper public static void AppendBytesToFile(string input, string output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) { // If any of the inputs are invalid, skip - if (!System.IO.File.Exists(input)) + if (!File.Exists(input)) { return; } - using (BinaryReader br = new BinaryReader(System.IO.File.OpenRead(input))) - using (BinaryWriter bw = new BinaryWriter(System.IO.File.OpenWrite(output))) + using (BinaryReader br = new BinaryReader(File.OpenRead(input))) + using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output))) { if (bytesToAddToHead.Count() > 0) { @@ -751,13 +751,13 @@ namespace SabreTools.Helper /// Output filename public static void CopyFileToNewLocation(string input, string output) { - if (System.IO.File.Exists(input) && !System.IO.File.Exists(output)) + if (File.Exists(input) && !File.Exists(output)) { if (!Directory.Exists(Path.GetDirectoryName(output))) { Directory.CreateDirectory(Path.GetDirectoryName(output)); } - System.IO.File.Copy(input, output); + File.Copy(input, output); } } } diff --git a/SabreTools.Helper/Tools/RomTools.cs b/SabreTools.Helper/Tools/RomTools.cs index 6cf5e885..55cab2cc 100644 --- a/SabreTools.Helper/Tools/RomTools.cs +++ b/SabreTools.Helper/Tools/RomTools.cs @@ -21,7 +21,7 @@ namespace SabreTools.Helper public static Rom GetSingleFileInfo(string input, bool noMD5 = false, bool noSHA1 = false, long offset = 0) { // Add safeguard if file doesn't exist - if (!System.IO.File.Exists(input)) + if (!File.Exists(input)) { return new Rom(); } @@ -44,7 +44,7 @@ namespace SabreTools.Helper using (Crc32 crc = new Crc32()) using (MD5 md5 = MD5.Create()) using (SHA1 sha1 = SHA1.Create()) - using (FileStream fs = System.IO.File.OpenRead(input)) + using (FileStream fs = File.OpenRead(input)) { // Seek to the starting position, if one is set if (offset > 0) @@ -186,6 +186,104 @@ namespace SabreTools.Helper return outroms; } + /// + /// Merge an arbitrary set of ROMs based on the supplied information + /// + /// List of HashData objects representing the roms to be merged + /// Logger object for console and/or file output + /// A List of HashData objects representing the merged roms + public static List Merge(List inhashes, Logger logger) + { + // Create output list + List outroms = new List(); + + // Check for null or blank roms first + if (inhashes == null || inhashes.Count == 0) + { + return outroms; + } + + // Then deduplicate them by checking to see if data matches previous saved roms + foreach (HashData hash in inhashes) + { + // If it's a nodump, add and skip + if (hash.Roms[0].Nodump) + { + outroms.Add(hash); + continue; + } + + // If it's the first rom in the list, don't touch it + if (outroms.Count != 0) + { + // Check if the rom is a duplicate + DupeType dupetype = DupeType.None; + HashData savedHash = new HashData(); + int pos = -1; + for (int i = 0; i < outroms.Count; i++) + { + HashData lastrom = outroms[i]; + RomData savedRom = savedHash.Roms[0]; + MachineData savedMachine = savedRom.Machine; + + // Get the duplicate status + dupetype = GetDuplicateStatus(hash, lastrom, logger); + + // If it's a duplicate, skip adding it to the output but add any missing information + if (dupetype != DupeType.None) + { + savedHash = lastrom; + pos = i; + + savedHash.CRC = (savedHash.CRC == null && hash.CRC != null ? hash.CRC : savedHash.CRC); + savedHash.MD5 = (savedHash.MD5 == null && hash.MD5 != null ? hash.MD5 : savedHash.MD5); + savedHash.SHA1 = (savedHash.SHA1 == null && hash.SHA1 != null ? hash.SHA1 : savedHash.SHA1); + savedRom.DupeType = dupetype; + + // If the current system has a lower ID than the previous, set the system accordingly + if (hash.Roms[0].Machine.SystemID < savedMachine.SystemID) + { + savedMachine.SystemID = hash.Roms[0].Machine.SystemID; + savedMachine.System = hash.Roms[0].Machine.System; + savedMachine.Name = hash.Roms[0].Machine.Name; + savedRom.Name = hash.Roms[0].Name; + } + + // If the current source has a lower ID than the previous, set the source accordingly + if (hash.Roms[0].Machine.SourceID < savedMachine.SourceID) + { + savedMachine.SourceID = hash.Roms[0].Machine.SourceID; + savedMachine.Source = hash.Roms[0].Machine.Source; + savedMachine.Name = hash.Roms[0].Machine.Name; + savedRom.Name = hash.Roms[0].Name; + } + + break; + } + } + + // If no duplicate is found, add it to the list + if (dupetype == DupeType.None) + { + outroms.Add(hash); + } + // Otherwise, if a new rom information is found, add that + else + { + outroms.RemoveAt(pos); + outroms.Insert(pos, savedHash); + } + } + else + { + outroms.Add(hash); + } + } + + // Then return the result + return outroms; + } + /// /// List all duplicates found in a DAT based on a rom /// @@ -348,5 +446,51 @@ namespace SabreTools.Helper return hash; } + + /// + /// Clean a hash byte array and pad to the correct size + /// + /// Hash byte array to sanitize + /// Amount of bytes to pad to + /// Cleaned byte array + public static byte[] CleanHashData(byte[] hash, int padding) + { + // If we have a null hash or a <=0 padding, return the hash + if (hash == null || padding <= 0) + { + return hash; + } + + // If we have a hash longer than the padding, trim and return + if (hash.Length > padding) + { + return hash.Take(padding).ToArray(); + } + + // If we have a hash of the correct length, return + if (hash.Length == padding) + { + return hash; + } + + // Otherwise get the output byte array of the correct length + byte[] newhash = new byte[padding]; + + // Then write the proper number of empty bytes + int padNeeded = padding - hash.Length; + int index = 0; + for (index = 0; index < padNeeded; index++) + { + newhash[index] = 0x00; + } + + // Now add the original hash + for (int i = 0; i < hash.Length; i++) + { + newhash[index + i] = hash[index]; + } + + return newhash; + } } } diff --git a/SabreTools/DATFromDir.cs b/SabreTools/DATFromDir.cs index e2ee39c0..519bc17f 100644 --- a/SabreTools/DATFromDir.cs +++ b/SabreTools/DATFromDir.cs @@ -71,7 +71,7 @@ namespace SabreTools public bool Start() { // Double check to see what it needs to be named - _basePath = (_inputs.Count > 0 ? (System.IO.File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : ""); + _basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : ""); _basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : ""); // If the description is defined but not the name, set the name from the description @@ -115,7 +115,7 @@ namespace SabreTools else { // Create and open the output file for writing - FileStream fs = System.IO.File.Create(Style.CreateOutfileName(Environment.CurrentDirectory, _datdata)); + FileStream fs = File.Create(Style.CreateOutfileName(Environment.CurrentDirectory, _datdata)); sw = new StreamWriter(fs, Encoding.UTF8); sw.AutoFlush = true; } @@ -128,11 +128,11 @@ namespace SabreTools foreach (string path in _inputs) { // Set local paths and vars - _basePath = (System.IO.File.Exists(path) ? path : path + Path.DirectorySeparatorChar); + _basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar); _basePath = Path.GetFullPath(_basePath); // This is where the main loop would go - if (System.IO.File.Exists(_basePath)) + if (File.Exists(_basePath)) { lastparent = ProcessPossibleArchive(_basePath, sw, lastparent); } @@ -152,7 +152,7 @@ namespace SabreTools { if (_datdata.Type != "SuperDAT") { - _basePath = (System.IO.File.Exists(item) ? item : item + Path.DirectorySeparatorChar); + _basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar); _basePath = Path.GetFullPath(_basePath); } @@ -170,7 +170,7 @@ namespace SabreTools if (!items) { string actualroot = item.Remove(0, basePathBackup.Length); - Helper.Rom rom = new Helper.Rom + Rom rom = new Rom { Name = "null", Machine = new Machine @@ -197,7 +197,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } @@ -209,7 +209,7 @@ namespace SabreTools if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0) { string actualroot = subdir.Remove(0, basePathBackup.Length); - Helper.Rom rom = new Helper.Rom + Rom rom = new Rom { Name = "null", Machine = new Machine @@ -236,7 +236,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } @@ -259,10 +259,10 @@ namespace SabreTools List keys = _datdata.Files.Keys.ToList(); foreach (string key in keys) { - List roms = _datdata.Files[key]; + List roms = _datdata.Files[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; // If we're in a mode that doesn't allow for actual empty folders, add the blank info if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile) @@ -284,7 +284,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _datdata.Files.Add(inkey, temp); } @@ -318,7 +318,7 @@ namespace SabreTools // If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting if (lastparent != null && _datdata.Files.Count == 0) { - _datdata.Files.Add("temp", new List()); + _datdata.Files.Add("temp", new List()); } } @@ -346,7 +346,7 @@ namespace SabreTools // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes) if (_datdata.Romba) { - Helper.Rom rom = ArchiveTools.GetTorrentGZFileInfo(item, _logger); + Rom rom = ArchiveTools.GetTorrentGZFileInfo(item, _logger); // If the rom is valid, write it out if (rom.Name != null) @@ -362,7 +362,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } @@ -391,8 +391,8 @@ namespace SabreTools // If we have an archive, scan it if (type != null) { - List extracted = ArchiveTools.GetArchiveFileInfo(item, _logger); - foreach (Helper.Rom rom in extracted) + List extracted = ArchiveTools.GetArchiveFileInfo(item, _logger); + foreach (Rom rom in extracted) { lastparent = ProcessFileHelper(item, rom, sw, _basePath, Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) + @@ -401,7 +401,7 @@ namespace SabreTools } } // Otherwise, just get the info on the file itself - else if (!Directory.Exists(item) && System.IO.File.Exists(item)) + else if (!Directory.Exists(item) && File.Exists(item)) { lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent); } @@ -440,7 +440,7 @@ namespace SabreTools } } // Otherwise, just get the info on the file itself - else if (!Directory.Exists(item) && System.IO.File.Exists(item)) + else if (!Directory.Exists(item) && File.Exists(item)) { lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent); } @@ -462,7 +462,7 @@ namespace SabreTools private string ProcessFile(string item, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent) { _logger.Log(Path.GetFileName(item) + " treated like a file"); - Helper.Rom rom = RomTools.GetSingleFileInfo(item, _noMD5, _noSHA1); + Rom rom = RomTools.GetSingleFileInfo(item, _noMD5, _noSHA1); return ProcessFileHelper(item, rom, sw, basepath, parent, datdata, lastparent); } @@ -478,7 +478,7 @@ namespace SabreTools /// DatData object with output information /// Last known parent game name /// New last known parent game name - private string ProcessFileHelper(string item, Helper.Rom rom, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent) + private string ProcessFileHelper(string item, Rom rom, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent) { try { @@ -537,7 +537,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } diff --git a/SabreTools/ImportExport/GenerateTwo.cs b/SabreTools/ImportExport/GenerateTwo.cs index 4a157fe0..249e8a6b 100644 --- a/SabreTools/ImportExport/GenerateTwo.cs +++ b/SabreTools/ImportExport/GenerateTwo.cs @@ -163,7 +163,7 @@ namespace SabreTools foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories)) { string hash = ""; - using (FileStream fs = System.IO.File.Open(file, FileMode.Open)) + using (FileStream fs = File.Open(file, FileMode.Open)) { hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", ""); } @@ -188,11 +188,11 @@ namespace SabreTools List keys = datdata.Files.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List newroms = datdata.Files[key]; + List temp = new List(); + List newroms = datdata.Files[key]; for (int i = 0; i < newroms.Count; i++) { - Helper.Rom rom = newroms[i]; + Rom rom = newroms[i]; // In the case that the RomData is incomplete, skip it if (rom.Name == null || rom.Machine.Name == null) diff --git a/SabreTools/OfflineMerge.cs b/SabreTools/OfflineMerge.cs index bae715d8..a457a9f2 100644 --- a/SabreTools/OfflineMerge.cs +++ b/SabreTools/OfflineMerge.cs @@ -39,15 +39,15 @@ namespace SabreTools public bool Process() { // Check all of the files for validity and break if one doesn't exist - if (_currentAllMerged != "" && !System.IO.File.Exists(_currentAllMerged)) + if (_currentAllMerged != "" && !File.Exists(_currentAllMerged)) { return false; } - if (_currentMissingMerged != "" && !System.IO.File.Exists(_currentMissingMerged)) + if (_currentMissingMerged != "" && !File.Exists(_currentMissingMerged)) { return false; } - if (_currentNewMerged != "" && !System.IO.File.Exists(_currentNewMerged)) + if (_currentNewMerged != "" && !File.Exists(_currentNewMerged)) { return false; } @@ -63,11 +63,11 @@ namespace SabreTools // Now get Net New output dictionary [(currentNewMerged)-(currentAllMerged)] _logger.User("Creating and populating Net New dictionary"); - Dictionary> netNew = new Dictionary>(); + Dictionary> netNew = new Dictionary>(); foreach (string key in completeDats.Files.Keys) { - List templist = RomTools.Merge(completeDats.Files[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(completeDats.Files[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged) { @@ -77,7 +77,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); netNew.Add(key, temp); } @@ -87,11 +87,11 @@ namespace SabreTools // Now create the Unneeded dictionary [(currentAllMerged)-(currentNewMerged)] _logger.User("Creating and populating Uneeded dictionary"); - Dictionary> unneeded = new Dictionary>(); + Dictionary> unneeded = new Dictionary>(); foreach (string key in completeDats.Files.Keys) { - List templist = RomTools.Merge(completeDats.Files[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(completeDats.Files[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged) { @@ -101,7 +101,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); unneeded.Add(key, temp); } @@ -124,11 +124,11 @@ namespace SabreTools midMissing.Files.Add(key, unneeded[key]); } } - Dictionary> newMissing = new Dictionary>(); + Dictionary> newMissing = new Dictionary>(); foreach (string key in midMissing.Files.Keys) { - List templist = RomTools.Merge(midMissing.Files[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(midMissing.Files[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentMissingMerged) { @@ -138,7 +138,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); newMissing.Add(key, temp); } @@ -159,7 +159,7 @@ namespace SabreTools // Now create the Have dictionary [(currentNewMerged)-(c)] _logger.User("Creating and populating Have dictionary"); - Dictionary> midHave = new Dictionary>(); + Dictionary> midHave = new Dictionary>(); foreach (string key in newMissing.Keys) { if (midHave.ContainsKey(key)) @@ -175,7 +175,7 @@ namespace SabreTools { if (midHave.ContainsKey(key)) { - foreach (Helper.Rom rom in completeDats.Files[key]) + foreach (Rom rom in completeDats.Files[key]) { if (rom.Metadata.System == _currentNewMerged) { @@ -185,8 +185,8 @@ namespace SabreTools } else { - List roms = new List(); - foreach (Helper.Rom rom in completeDats.Files[key]) + List roms = new List(); + foreach (Rom rom in completeDats.Files[key]) { if (rom.Metadata.System == _currentNewMerged) { @@ -196,11 +196,11 @@ namespace SabreTools midHave.Add(key, roms); } } - Dictionary> have = new Dictionary>(); + Dictionary> have = new Dictionary>(); foreach (string key in midHave.Keys) { - List templist = RomTools.Merge(midHave[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(midHave[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged) { @@ -210,7 +210,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); have.Add(key, temp); } @@ -225,11 +225,11 @@ namespace SabreTools List keys = netNew.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = netNew[key]; + List temp = new List(); + List roms = netNew[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; @@ -243,11 +243,11 @@ namespace SabreTools keys = unneeded.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = unneeded[key]; + List temp = new List(); + List roms = unneeded[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; @@ -261,11 +261,11 @@ namespace SabreTools keys = newMissing.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = newMissing[key]; + List temp = new List(); + List roms = newMissing[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; @@ -279,11 +279,11 @@ namespace SabreTools keys = have.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = have[key]; + List temp = new List(); + List roms = have[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; @@ -364,11 +364,11 @@ namespace SabreTools Dat midHave = new Dat(); midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger); midHave = DatTools.Parse(_currentAllMerged, 0, 0, midHave, _logger); - Dictionary> have = new Dictionary>(); + Dictionary> have = new Dictionary>(); foreach (string key in midHave.Files.Keys) { - List templist = RomTools.Merge(midHave.Files[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(midHave.Files[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged) { @@ -378,7 +378,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); have.Add(key, temp); } @@ -393,11 +393,11 @@ namespace SabreTools List keys = have.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = have[key]; + List temp = new List(); + List roms = have[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; @@ -434,11 +434,11 @@ namespace SabreTools Dat midHave = new Dat(); midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger); midHave = DatTools.Parse(_currentNewMerged, 0, 0, midHave, _logger); - Dictionary> have = new Dictionary>(); + Dictionary> have = new Dictionary>(); foreach (string key in midHave.Files.Keys) { - List templist = RomTools.Merge(midHave.Files[key], _logger); - foreach (Helper.Rom rom in templist) + List templist = RomTools.Merge(midHave.Files[key], _logger); + foreach (Rom rom in templist) { if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged) { @@ -448,7 +448,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); have.Add(key, temp); } @@ -463,11 +463,11 @@ namespace SabreTools List keys = have.Keys.ToList(); foreach (string key in keys) { - List temp = new List(); - List roms = have[key]; + List temp = new List(); + List roms = have[key]; for (int i = 0; i < roms.Count; i++) { - Helper.Rom rom = roms[i]; + Rom rom = roms[i]; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index 91f6c4e7..43f6cdac 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -411,7 +411,7 @@ namespace SabreTools OutputFormat = (old ? OutputFormat.ClrMamePro : OutputFormat.Xml), Romba = romba, Type = (superdat ? "SuperDAT" : ""), - Files = new Dictionary>(), + Files = new Dictionary>(), }; DATFromDir dfd = new DATFromDir(inputs, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, _logger); bool success = dfd.Start(); @@ -436,7 +436,7 @@ namespace SabreTools // Verify the input files foreach (string input in inputs) { - if (!System.IO.File.Exists(input) && !Directory.Exists(input)) + if (!File.Exists(input) && !Directory.Exists(input)) { _logger.Error(input + " is not a valid file or folder!"); Console.WriteLine(); @@ -464,7 +464,7 @@ namespace SabreTools // Verify the input files foreach (string input in inputs) { - if (!System.IO.File.Exists(input) && !Directory.Exists(input)) + if (!File.Exists(input) && !Directory.Exists(input)) { _logger.Error(input + " is not a valid file or folder!"); Console.WriteLine(); @@ -506,7 +506,7 @@ namespace SabreTools foreach (string input in inputs) { - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { newinputs.Add(input); } diff --git a/SabreTools/Split.cs b/SabreTools/Split.cs index 08bd3989..e3c08bd2 100644 --- a/SabreTools/Split.cs +++ b/SabreTools/Split.cs @@ -80,7 +80,7 @@ namespace SabreTools foreach (string input in _inputs) { // If it's a file, run the proper split on the file - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { if (_hash) { @@ -155,7 +155,7 @@ namespace SabreTools ForcePacking = datdata.ForcePacking, OutputFormat = outputFormat, MergeRoms = datdata.MergeRoms, - Files = new Dictionary>(), + Files = new Dictionary>(), }; Dat sha1 = new Dat { @@ -177,7 +177,7 @@ namespace SabreTools ForcePacking = datdata.ForcePacking, OutputFormat = outputFormat, MergeRoms = datdata.MergeRoms, - Files = new Dictionary>(), + Files = new Dictionary>(), }; Dat md5 = new Dat { @@ -199,7 +199,7 @@ namespace SabreTools ForcePacking = datdata.ForcePacking, OutputFormat = outputFormat, MergeRoms = datdata.MergeRoms, - Files = new Dictionary>(), + Files = new Dictionary>(), }; Dat crc = new Dat { @@ -221,15 +221,15 @@ namespace SabreTools ForcePacking = datdata.ForcePacking, OutputFormat = outputFormat, MergeRoms = datdata.MergeRoms, - Files = new Dictionary>(), + Files = new Dictionary>(), }; // Now populate each of the DAT objects in turn List keys = datdata.Files.Keys.ToList(); foreach (string key in keys) { - List roms = datdata.Files[key]; - foreach (Helper.Rom rom in roms) + List roms = datdata.Files[key]; + foreach (Rom rom in roms) { // If the file is a nodump if (rom.Nodump) @@ -240,7 +240,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); nodump.Files.Add(key, temp); } @@ -254,7 +254,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); sha1.Files.Add(key, temp); } @@ -268,7 +268,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); md5.Files.Add(key, temp); } @@ -282,7 +282,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); crc.Files.Add(key, temp); } @@ -351,7 +351,7 @@ namespace SabreTools Homepage = datdata.Homepage, Url = datdata.Url, Comment = datdata.Comment, - Files = new Dictionary>(), + Files = new Dictionary>(), OutputFormat = outputFormat, }; Dat datdataB = new Dat @@ -367,7 +367,7 @@ namespace SabreTools Homepage = datdata.Homepage, Url = datdata.Url, Comment = datdata.Comment, - Files = new Dictionary>(), + Files = new Dictionary>(), OutputFormat = outputFormat, }; @@ -380,7 +380,7 @@ namespace SabreTools // Now separate the roms accordingly foreach (string key in datdata.Files.Keys) { - foreach (Helper.Rom rom in datdata.Files[key]) + foreach (Rom rom in datdata.Files[key]) { if (_extA.Contains(Path.GetExtension(rom.Name.ToUpperInvariant()))) { @@ -390,7 +390,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); datdataA.Files.Add(key, temp); } @@ -403,7 +403,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); datdataB.Files.Add(key, temp); } @@ -416,7 +416,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); datdataA.Files.Add(key, temp); } @@ -426,7 +426,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); datdataB.Files.Add(key, temp); } diff --git a/SimpleSort/SimpleSort.cs b/SimpleSort/SimpleSort.cs index afe0a18a..c14f5853 100644 --- a/SimpleSort/SimpleSort.cs +++ b/SimpleSort/SimpleSort.cs @@ -69,7 +69,7 @@ namespace SabreTools _cursorLeft = Console.CursorLeft; _matched = new Dat { - Files = new Dictionary>(), + Files = new Dictionary>(), }; } @@ -167,7 +167,7 @@ namespace SabreTools else if (temparg.StartsWith("-dat=") || temparg.StartsWith("--dat=")) { string datfile = temparg.Split('=')[1]; - if (!System.IO.File.Exists(datfile)) + if (!File.Exists(datfile)) { logger.Error("DAT must be a valid file: " + datfile); Console.WriteLine(); @@ -206,7 +206,7 @@ namespace SabreTools zip = 0; } } - else if (System.IO.File.Exists(temparg) || Directory.Exists(temparg)) + else if (File.Exists(temparg) || Directory.Exists(temparg)) { inputs.Add(temparg); } @@ -358,7 +358,7 @@ namespace SabreTools // Setup the fixdat _matched = (Dat)_datdata.CloneHeader(); - _matched.Files = new Dictionary>(); + _matched.Files = new Dictionary>(); _matched.FileName = "fixDat_" + _matched.FileName; _matched.Name = "fixDat_" + _matched.Name; _matched.Description = "fixDat_" + _matched.Description; @@ -366,10 +366,10 @@ namespace SabreTools // Now that all files are parsed, get only files found in directory bool found = false; - foreach (List roms in _datdata.Files.Values) + foreach (List roms in _datdata.Files.Values) { - List newroms = RomTools.Merge(roms, _logger); - foreach (Helper.Rom rom in newroms) + List newroms = RomTools.Merge(roms, _logger); + foreach (Rom rom in newroms) { if (rom.Metadata.SourceID == 99) { @@ -381,7 +381,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(rom); _matched.Files.Add(key, temp); } @@ -414,7 +414,7 @@ namespace SabreTools List files = new List(); foreach (string input in _inputs) { - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { _logger.Log("File found: '" + input + "'"); files.Add(Path.GetFullPath(input)); @@ -492,7 +492,7 @@ namespace SabreTools // Hash and match the external files if (shouldExternalScan) { - Helper.Rom rom = RomTools.GetSingleFileInfo(input); + Rom rom = RomTools.GetSingleFileInfo(input); // If we have a blank RomData, it's an error if (rom.Name == null) @@ -501,9 +501,9 @@ namespace SabreTools } // Try to find the matches to the file that was found - List foundroms = RomTools.GetDuplicates(rom, _datdata, _logger, true); + List foundroms = RomTools.GetDuplicates(rom, _datdata, _logger, true); _logger.Log("File '" + input + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Helper.Rom found in foundroms) + foreach (Rom found in foundroms) { _logger.Log("Matched name: " + found.Name); @@ -515,7 +515,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(found); _matched.Files.Add(key, temp); } @@ -532,7 +532,7 @@ namespace SabreTools _logger.Log("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.HashData.SHA1 : found.Name) + "'"); try { - System.IO.File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name))); + File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name))); } catch { } } @@ -558,7 +558,7 @@ namespace SabreTools // Otherwise, apply the rule ot the file string newinput = input + ".new"; Skippers.TransformFile(input, newinput, rule, _logger); - Helper.Rom drom = RomTools.GetSingleFileInfo(newinput); + Rom drom = RomTools.GetSingleFileInfo(newinput); // If we have a blank RomData, it's an error if (drom.Name == null) @@ -567,9 +567,9 @@ namespace SabreTools } // Try to find the matches to the file that was found - List founddroms = RomTools.GetDuplicates(drom, _datdata, _logger, true); + List founddroms = RomTools.GetDuplicates(drom, _datdata, _logger, true); _logger.Log("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!"); - foreach (Helper.Rom found in founddroms) + foreach (Rom found in founddroms) { // Add rom to the matched list string key = found.HashData.Size + "-" + found.HashData.CRC; @@ -579,7 +579,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(found); _matched.Files.Add(key, temp); } @@ -599,7 +599,7 @@ namespace SabreTools _logger.Log("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.HashData.SHA1 : found.Name) + "'"); try { - System.IO.File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name))); + File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name))); } catch { } } @@ -616,7 +616,7 @@ namespace SabreTools } // Then output the headered rom (renamed) - Helper.Rom newfound = found; + Rom newfound = found; newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.HashData.CRC + ")" + Path.GetExtension(newfound.Name); // Add rom to the matched list @@ -627,7 +627,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(newfound); _matched.Files.Add(key, temp); } @@ -644,7 +644,7 @@ namespace SabreTools _logger.Log("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'"); try { - System.IO.File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name))); + File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name))); } catch { } } @@ -665,7 +665,7 @@ namespace SabreTools // Now remove this temporary file try { - System.IO.File.Delete(newinput); + File.Delete(newinput); } catch { @@ -681,18 +681,18 @@ namespace SabreTools if (_quickScan) { _logger.Log("Beginning quick scan of contents from '" + input + "'"); - List internalRomData = ArchiveTools.GetArchiveFileInfo(input, _logger); + List internalRomData = ArchiveTools.GetArchiveFileInfo(input, _logger); _logger.Log(internalRomData.Count + " entries found in '" + input + "'"); // If the list is populated, then the file was a filled archive if (internalRomData.Count > 0) { - foreach (Helper.Rom rom in internalRomData) + foreach (Rom rom in internalRomData) { // Try to find the matches to the file that was found - List foundroms = RomTools.GetDuplicates(rom, _datdata, _logger, true); + List foundroms = RomTools.GetDuplicates(rom, _datdata, _logger, true); _logger.Log("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Helper.Rom found in foundroms) + foreach (Rom found in foundroms) { // Add rom to the matched list string key = found.HashData.Size + "-" + found.HashData.CRC; @@ -702,7 +702,7 @@ namespace SabreTools } else { - List temp = new List(); + List temp = new List(); temp.Add(found); _matched.Files.Add(key, temp); } @@ -712,7 +712,7 @@ namespace SabreTools // Copy file to output directory _logger.Log("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'"); string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempdir, _logger); - if (System.IO.File.Exists(outfile)) + if (File.Exists(outfile)) { string gamedir = Path.Combine(_outdir, found.Machine.Name); if (!Directory.Exists(gamedir)) @@ -722,7 +722,7 @@ namespace SabreTools try { - System.IO.File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name))); + File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name))); } catch { } } @@ -735,7 +735,7 @@ namespace SabreTools if (Build.MonoEnvironment || _tgz) { string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempdir, _logger); - if (System.IO.File.Exists(outfile)) + if (File.Exists(outfile)) { if (_tgz) { @@ -748,7 +748,7 @@ namespace SabreTools try { - System.IO.File.Delete(outfile); + File.Delete(outfile); } catch { } } @@ -773,7 +773,7 @@ namespace SabreTools { try { - System.IO.File.Delete(input); + File.Delete(input); } catch (Exception) { @@ -825,11 +825,11 @@ namespace SabreTools } // Now process the inputs (assumed that it's archived sets as of right now - Dictionary> scanned = new Dictionary>(); + Dictionary> scanned = new Dictionary>(); foreach (string archive in Directory.EnumerateFiles(_outdir, "*", SearchOption.AllDirectories)) { // If we are in quickscan, get the list of roms that way - List roms = new List(); + List roms = new List(); if (_quickScan) { roms = ArchiveTools.GetArchiveFileInfo(Path.GetFullPath(archive), _logger); @@ -849,7 +849,7 @@ namespace SabreTools } // Then add each of the found files to the new dictionary - foreach (Helper.Rom rom in roms) + foreach (Rom rom in roms) { string key = rom.HashData.Size + "-" + rom.HashData.CRC; if (scanned.ContainsKey(key)) @@ -858,7 +858,7 @@ namespace SabreTools } else { - List templist = new List(); + List templist = new List(); templist.Add(rom); scanned.Add(key, templist); } @@ -872,7 +872,7 @@ namespace SabreTools } // Now that we have all of the from DAT and from folder roms, we try to match them, removing the perfect matches - Dictionary> remove = new Dictionary>(); + Dictionary> remove = new Dictionary>(); foreach (string key in scanned.Keys) { // If the key doesn't even exist in the DAT, then mark the entire key for removal @@ -890,9 +890,9 @@ namespace SabreTools // Otherwise check each of the values individually else { - List romsList = _datdata.Files[key]; - List scannedList = scanned[key]; - foreach (Helper.Rom rom in scannedList) + List romsList = _datdata.Files[key]; + List scannedList = scanned[key]; + foreach (Rom rom in scannedList) { if (!romsList.Contains(rom)) { @@ -902,7 +902,7 @@ namespace SabreTools } else { - List templist = new List(); + List templist = new List(); templist.Add(rom); remove.Add(key, templist); } diff --git a/TGZTest/TGZTest.cs b/TGZTest/TGZTest.cs index d867d79d..afa95e6d 100644 --- a/TGZTest/TGZTest.cs +++ b/TGZTest/TGZTest.cs @@ -152,7 +152,7 @@ namespace SabreTools zip = 0; } } - else if (System.IO.File.Exists(temparg) || Directory.Exists(temparg)) + else if (File.Exists(temparg) || Directory.Exists(temparg)) { inputs.Add(temparg); } @@ -223,7 +223,7 @@ namespace SabreTools List newinputs = new List(); foreach (string input in inputs) { - if (System.IO.File.Exists(input)) + if (File.Exists(input)) { newinputs.Add(Path.GetFullPath(input)); } @@ -305,7 +305,7 @@ namespace SabreTools try { _logger.User("Attempting to delete " + input); - System.IO.File.Delete(input); + File.Delete(input); } catch (Exception ex) { @@ -329,7 +329,7 @@ namespace SabreTools } // If we're in romba mode and the size file doesn't exist, create it - if (_romba && !System.IO.File.Exists(Path.Combine(_outdir, ".romba_size"))) + if (_romba && !File.Exists(Path.Combine(_outdir, ".romba_size"))) { // Get the size of all of the files in the output folder long size = 0; @@ -340,8 +340,8 @@ namespace SabreTools } // Write out the value to each of the romba depot files - using (StreamWriter tw = new StreamWriter(System.IO.File.Open(Path.Combine(_outdir, ".romba_size"), FileMode.Create, FileAccess.Write))) - using (StreamWriter twb = new StreamWriter(System.IO.File.Open(Path.Combine(_outdir, ".romba_size.backup"), FileMode.Create, FileAccess.Write))) + using (StreamWriter tw = new StreamWriter(File.Open(Path.Combine(_outdir, ".romba_size"), FileMode.Create, FileAccess.Write))) + using (StreamWriter twb = new StreamWriter(File.Open(Path.Combine(_outdir, ".romba_size.backup"), FileMode.Create, FileAccess.Write))) { tw.Write(size); twb.Write(size);