From 1e4afb30465766b76eed337184dd56d5489d9429 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 17 Sep 2020 23:07:49 -0700 Subject: [PATCH] Decouple stream hashing from special file info --- SabreTools.Library/DatFiles/AttractMode.cs | 2 +- SabreTools.Library/DatFiles/ClrMamePro.cs | 1 - SabreTools.Library/DatFiles/DatFile.cs | 2 +- SabreTools.Library/FileTypes/BaseFile.cs | 32 +++++++- SabreTools.Library/FileTypes/GZipArchive.cs | 3 +- SabreTools.Library/FileTypes/RarArchive.cs | 3 +- .../FileTypes/SevenZipArchive.cs | 3 +- SabreTools.Library/FileTypes/TapeArchive.cs | 3 +- SabreTools.Library/FileTypes/XZArchive.cs | 3 +- SabreTools.Library/FileTypes/ZipArchive.cs | 3 +- SabreTools.Library/IO/FileExtensions.cs | 39 ++++++++-- SabreTools.Library/IO/StreamExtensions.cs | 78 ++++--------------- 12 files changed, 93 insertions(+), 79 deletions(-) diff --git a/SabreTools.Library/DatFiles/AttractMode.cs b/SabreTools.Library/DatFiles/AttractMode.cs index 131c6fcb..cb181f84 100644 --- a/SabreTools.Library/DatFiles/AttractMode.cs +++ b/SabreTools.Library/DatFiles/AttractMode.cs @@ -62,7 +62,7 @@ namespace SabreTools.Library.DatFiles } catch (InvalidDataException ex) { - Globals.Logger.Warning($"Malformed line found in '{filename}' at line {svr.LineNumber}"); + Globals.Logger.Error(ex, $"Malformed line found in '{filename}' at line {svr.LineNumber}"); if (throwOnError) throw ex; continue; } diff --git a/SabreTools.Library/DatFiles/ClrMamePro.cs b/SabreTools.Library/DatFiles/ClrMamePro.cs index 985ab3d3..260db0ad 100644 --- a/SabreTools.Library/DatFiles/ClrMamePro.cs +++ b/SabreTools.Library/DatFiles/ClrMamePro.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; using SabreTools.Library.Data; diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index 779c867d..5d8dae2e 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -1867,7 +1867,7 @@ namespace SabreTools.Library.DatFiles } catch (Exception ex) { - Globals.Logger.Error($"Error with file '{filename}': {ex}"); + Globals.Logger.Error(ex, $"Error with file '{filename}'"); if (throwOnError) throw ex; } } diff --git a/SabreTools.Library/FileTypes/BaseFile.cs b/SabreTools.Library/FileTypes/BaseFile.cs index 5ae04185..7d4220c8 100644 --- a/SabreTools.Library/FileTypes/BaseFile.cs +++ b/SabreTools.Library/FileTypes/BaseFile.cs @@ -1,5 +1,5 @@ using System.IO; - +using SabreTools.Library.DatFiles; using SabreTools.Library.IO; namespace SabreTools.Library.FileTypes @@ -99,5 +99,35 @@ namespace SabreTools.Library.FileTypes } #endregion + + #region Manipulation + + /// + /// Remove hashes from the metadata + /// + /// Hash representing hashes to remove + public void RemoveHashes(Hash remove) + { + if (remove.HasFlag(Hash.CRC)) + CRC = null; + if (remove.HasFlag(Hash.MD5)) + MD5 = null; +#if NET_FRAMEWORK + if (remove.HasFlag(Hash.RIPEMD160)) + RIPEMD160 = null; +#endif + if (remove.HasFlag(Hash.SHA1)) + SHA1 = null; + if (remove.HasFlag(Hash.SHA256)) + SHA256 = null; + if (remove.HasFlag(Hash.SHA384)) + SHA384 = null; + if (remove.HasFlag(Hash.SHA512)) + SHA512 = null; + if (remove.HasFlag(Hash.SpamSum)) + SpamSum = null; + } + + #endregion } } diff --git a/SabreTools.Library/FileTypes/GZipArchive.cs b/SabreTools.Library/FileTypes/GZipArchive.cs index edc298c3..330b6de5 100644 --- a/SabreTools.Library/FileTypes/GZipArchive.cs +++ b/SabreTools.Library/FileTypes/GZipArchive.cs @@ -233,7 +233,8 @@ namespace SabreTools.Library.FileTypes var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); - BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile gzipEntryRom = gzstream.GetInfo(); + gzipEntryRom.RemoveHashes(omitFromScan); gzipEntryRom.Filename = gz.Filename(0); gzipEntryRom.Parent = gamename; gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null); diff --git a/SabreTools.Library/FileTypes/RarArchive.cs b/SabreTools.Library/FileTypes/RarArchive.cs index e6b625c3..a3ba9481 100644 --- a/SabreTools.Library/FileTypes/RarArchive.cs +++ b/SabreTools.Library/FileTypes/RarArchive.cs @@ -206,7 +206,8 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size); + rarEntryRom.RemoveHashes(omitFromScan); rarEntryRom.Filename = entry.Key; rarEntryRom.Parent = gamename; rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/SevenZipArchive.cs b/SabreTools.Library/FileTypes/SevenZipArchive.cs index cdd72d13..ff5a1bfa 100644 --- a/SabreTools.Library/FileTypes/SevenZipArchive.cs +++ b/SabreTools.Library/FileTypes/SevenZipArchive.cs @@ -318,7 +318,8 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true); + zipEntryRom.RemoveHashes(omitFromScan); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; found.Add(zipEntryRom); diff --git a/SabreTools.Library/FileTypes/TapeArchive.cs b/SabreTools.Library/FileTypes/TapeArchive.cs index 2a5f3a87..9a2dfebd 100644 --- a/SabreTools.Library/FileTypes/TapeArchive.cs +++ b/SabreTools.Library/FileTypes/TapeArchive.cs @@ -211,7 +211,8 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size); + tarEntryRom.RemoveHashes(omitFromScan); tarEntryRom.Filename = entry.Key; tarEntryRom.Parent = gamename; tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/XZArchive.cs b/SabreTools.Library/FileTypes/XZArchive.cs index c9ba5b6c..8fac3f5c 100644 --- a/SabreTools.Library/FileTypes/XZArchive.cs +++ b/SabreTools.Library/FileTypes/XZArchive.cs @@ -223,7 +223,8 @@ namespace SabreTools.Library.FileTypes else { var xzStream = new XZStream(File.OpenRead(this.Filename)); - BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile xzEntryRom = xzStream.GetInfo(); + xzEntryRom.RemoveHashes(omitFromScan); xzEntryRom.Filename = gamename; xzEntryRom.Parent = gamename; _children.Add(xzEntryRom); diff --git a/SabreTools.Library/FileTypes/ZipArchive.cs b/SabreTools.Library/FileTypes/ZipArchive.cs index c51e83dc..ad668890 100644 --- a/SabreTools.Library/FileTypes/ZipArchive.cs +++ b/SabreTools.Library/FileTypes/ZipArchive.cs @@ -321,7 +321,8 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); + BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true); + zipEntryRom.RemoveHashes(omitFromScan); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; string convertedDate = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/IO/FileExtensions.cs b/SabreTools.Library/IO/FileExtensions.cs index a57ab421..ad3f3724 100644 --- a/SabreTools.Library/IO/FileExtensions.cs +++ b/SabreTools.Library/IO/FileExtensions.cs @@ -350,6 +350,10 @@ namespace SabreTools.Library.IO if (!File.Exists(input)) return null; + // Get input information + var fileType = input.GetFileType(); + var inputStream = TryOpenRead(input); + // Get the information from the file stream BaseFile baseFile; if (header != null) @@ -359,30 +363,49 @@ namespace SabreTools.Library.IO // If there's a match, get the new information from the stream if (rule.Tests != null && rule.Tests.Count != 0) { - // Create the input and output streams + // Create the output stream MemoryStream outputStream = new MemoryStream(); - FileStream inputStream = TryOpenRead(input); - + // Transform the stream and get the information from it rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true); - baseFile = outputStream.GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles); - // Dispose of the streams + if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats)) + baseFile = AaruFormat.Create(outputStream); + else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs)) + baseFile = CHDFile.Create(outputStream); + else + baseFile = outputStream.GetInfo(keepReadOpen: false); + + // Dispose of the output stream outputStream.Dispose(); - inputStream.Dispose(); + } // Otherwise, just get the info else { - baseFile = TryOpenRead(input).GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles); + if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats)) + baseFile = AaruFormat.Create(inputStream); + else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs)) + baseFile = CHDFile.Create(inputStream); + else + baseFile = inputStream.GetInfo(keepReadOpen: false); } } else { - baseFile = TryOpenRead(input).GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles); + if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats)) + baseFile = AaruFormat.Create(inputStream); + else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs)) + baseFile = CHDFile.Create(inputStream); + else + baseFile = inputStream.GetInfo(keepReadOpen: false); } + // Dispose of the input stream + inputStream.Dispose(); + // Add unique data from the file + baseFile.RemoveHashes(omitFromScan); baseFile.Filename = Path.GetFileName(input); baseFile.Date = (date ? new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss") : string.Empty); diff --git a/SabreTools.Library/IO/StreamExtensions.cs b/SabreTools.Library/IO/StreamExtensions.cs index 28dbb12e..b589c74c 100644 --- a/SabreTools.Library/IO/StreamExtensions.cs +++ b/SabreTools.Library/IO/StreamExtensions.cs @@ -45,11 +45,10 @@ namespace SabreTools.Library.IO /// Size of the input stream /// Hash flag saying what hashes should not be calculated (defaults to none) /// True if the underlying read stream should be kept open, false otherwise - /// TreatAsFiles representing special format scanning /// Populated BaseFile object if success, empty one on error - public static BaseFile GetInfo(this Stream input, long size = -1, Hash omitFromScan = 0x0, bool keepReadOpen = false, TreatAsFiles asFiles = 0x00) + public static BaseFile GetInfo(this Stream input, long size = -1, bool keepReadOpen = false) { - return GetInfoAsync(input, size, omitFromScan, keepReadOpen, asFiles).ConfigureAwait(false).GetAwaiter().GetResult(); + return GetInfoAsync(input, size, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult(); } /// @@ -57,53 +56,14 @@ namespace SabreTools.Library.IO /// /// Filename to get information from /// Size of the input stream - /// Hash flag saying what hashes should not be calculated (defaults to none) /// True if the underlying read stream should be kept open, false otherwise - /// TreatAsFiles representing special format scanning /// Populated BaseFile object if success, empty one on error - public static async Task GetInfoAsync(Stream input, long size = -1, Hash omitFromScan = 0x0, bool keepReadOpen = false, TreatAsFiles asFiles = 0x00) + public static async Task GetInfoAsync(Stream input, long size = -1, bool keepReadOpen = false) { // If we want to automatically set the size if (size == -1) size = input.Length; - // We first check to see if it's an AaruFormat if we have to - if (!asFiles.HasFlag(TreatAsFiles.AaruFormats)) - { - var aaruFormat = AaruFormat.Create(input); - input.SeekIfPossible(); - - // If we found a valid AaruFormat - if (aaruFormat != null) - { - if (!keepReadOpen) - input.Dispose(); - - return aaruFormat; - } - } - - // Then, we first check to see if it's a CHD if we have to - if (!asFiles.HasFlag(TreatAsFiles.CHDs)) - { - var chd = CHDFile.Create(input); - input.SeekIfPossible(); - - // If we found a valid CHD - if (chd != null) - { - if (!keepReadOpen) - input.Dispose(); - - return chd; - } - } - - BaseFile rom = new BaseFile() - { - Size = size, - }; - try { // Get a list of hashers to run over the buffer @@ -169,28 +129,26 @@ namespace SabreTools.Library.IO await Task.WhenAll(hashers.Select(h => h.Finalize())); // Get the results - if (!omitFromScan.HasFlag(Hash.CRC)) - rom.CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash(); - if (!omitFromScan.HasFlag(Hash.MD5)) - rom.MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash(); + BaseFile baseFile = new BaseFile() + { + Size = size, + CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash(), + MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash(), #if NET_FRAMEWORK - if (!omitFromScan.HasFlag(Hash.RIPEMD160)) - rom.RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash(); + RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash(), #endif - if (!omitFromScan.HasFlag(Hash.SHA1)) - rom.SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash(); - if (!omitFromScan.HasFlag(Hash.SHA256)) - rom.SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash(); - if (!omitFromScan.HasFlag(Hash.SHA384)) - rom.SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash(); - if (!omitFromScan.HasFlag(Hash.SHA512)) - rom.SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash(); - if (!omitFromScan.HasFlag(Hash.SpamSum)) - rom.SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash(); + SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash(), + SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash(), + SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash(), + SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash(), + SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash(), + }; // Dispose of the hashers loadBuffer.Dispose(); hashers.ForEach(h => h.Dispose()); + + return baseFile; } catch (IOException ex) { @@ -204,8 +162,6 @@ namespace SabreTools.Library.IO else input.SeekIfPossible(); } - - return rom; } ///