claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Decouple stream hashing from special file info

Browse Source
This commit is contained in:
Matt Nadareski
2020-09-17 23:07:49 -07:00
parent 3ad55d806b
commit 1e4afb3046
12 changed files with 93 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
SabreTools.Library/DatFiles/AttractMode.cs
View File

@@ -62,7 +62,7 @@ namespace SabreTools.Library.DatFiles
}
catch (InvalidDataException ex)
{
Globals.Logger.Warning($"Malformed line found in '{filename}' at line {svr.LineNumber}");
Globals.Logger.Error(ex, $"Malformed line found in '{filename}' at line {svr.LineNumber}");
if (throwOnError) throw ex;
continue;
}

1
SabreTools.Library/DatFiles/ClrMamePro.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using SabreTools.Library.Data;

2
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -1867,7 +1867,7 @@ namespace SabreTools.Library.DatFiles
}
catch (Exception ex)
{
Globals.Logger.Error($"Error with file '{filename}': {ex}");
Globals.Logger.Error(ex, $"Error with file '{filename}'");
if (throwOnError) throw ex;
}
}

32
SabreTools.Library/FileTypes/BaseFile.cs
View File

@@ -1,5 +1,5 @@
using System.IO;
using SabreTools.Library.DatFiles;
using SabreTools.Library.IO;
namespace SabreTools.Library.FileTypes
@@ -99,5 +99,35 @@ namespace SabreTools.Library.FileTypes
}
#endregion
#region Manipulation
/// <summary>
/// Remove hashes from the metadata
/// </summary>
/// <param name="remove">Hash representing hashes to remove</param>
public void RemoveHashes(Hash remove)
{
if (remove.HasFlag(Hash.CRC))
CRC = null;
if (remove.HasFlag(Hash.MD5))
MD5 = null;
#if NET_FRAMEWORK
if (remove.HasFlag(Hash.RIPEMD160))
RIPEMD160 = null;
#endif
if (remove.HasFlag(Hash.SHA1))
SHA1 = null;
if (remove.HasFlag(Hash.SHA256))
SHA256 = null;
if (remove.HasFlag(Hash.SHA384))
SHA384 = null;
if (remove.HasFlag(Hash.SHA512))
SHA512 = null;
if (remove.HasFlag(Hash.SpamSum))
SpamSum = null;
}
#endregion
}
}

3
SabreTools.Library/FileTypes/GZipArchive.cs
View File

@@ -233,7 +233,8 @@ namespace SabreTools.Library.FileTypes
var gz = new gZip();
ZipReturn ret = gz.ZipFileOpen(this.Filename);
ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile gzipEntryRom = gzstream.GetInfo();
gzipEntryRom.RemoveHashes(omitFromScan);
gzipEntryRom.Filename = gz.Filename(0);
gzipEntryRom.Parent = gamename;
gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);

3
SabreTools.Library/FileTypes/RarArchive.cs
View File

@@ -206,7 +206,8 @@ namespace SabreTools.Library.FileTypes
else
{
Stream entryStream = entry.OpenEntryStream();
BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size);
rarEntryRom.RemoveHashes(omitFromScan);
rarEntryRom.Filename = entry.Key;
rarEntryRom.Parent = gamename;
rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss");

3
SabreTools.Library/FileTypes/SevenZipArchive.cs
View File

@@ -318,7 +318,8 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true);
zipEntryRom.RemoveHashes(omitFromScan);
zipEntryRom.Filename = zf.Filename(i);
zipEntryRom.Parent = gamename;
found.Add(zipEntryRom);

3
SabreTools.Library/FileTypes/TapeArchive.cs
View File

@@ -211,7 +211,8 @@ namespace SabreTools.Library.FileTypes
else
{
Stream entryStream = entry.OpenEntryStream();
BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size);
tarEntryRom.RemoveHashes(omitFromScan);
tarEntryRom.Filename = entry.Key;
tarEntryRom.Parent = gamename;
tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss");

3
SabreTools.Library/FileTypes/XZArchive.cs
View File

@@ -223,7 +223,8 @@ namespace SabreTools.Library.FileTypes
else
{
var xzStream = new XZStream(File.OpenRead(this.Filename));
BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile xzEntryRom = xzStream.GetInfo();
xzEntryRom.RemoveHashes(omitFromScan);
xzEntryRom.Filename = gamename;
xzEntryRom.Parent = gamename;
_children.Add(xzEntryRom);

3
SabreTools.Library/FileTypes/ZipArchive.cs
View File

@@ -321,7 +321,8 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true);
zipEntryRom.RemoveHashes(omitFromScan);
zipEntryRom.Filename = zf.Filename(i);
zipEntryRom.Parent = gamename;
string convertedDate = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss");

37
SabreTools.Library/IO/FileExtensions.cs
View File

@@ -350,6 +350,10 @@ namespace SabreTools.Library.IO
if (!File.Exists(input))
return null;
// Get input information
var fileType = input.GetFileType();
var inputStream = TryOpenRead(input);
// Get the information from the file stream
BaseFile baseFile;
if (header != null)
@@ -359,30 +363,49 @@ namespace SabreTools.Library.IO
// If there's a match, get the new information from the stream
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the input and output streams
// Create the output stream
MemoryStream outputStream = new MemoryStream();
FileStream inputStream = TryOpenRead(input);
// Transform the stream and get the information from it
rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true);
baseFile = outputStream.GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles);
// Dispose of the streams
if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats))
baseFile = AaruFormat.Create(outputStream);
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs))
baseFile = CHDFile.Create(outputStream);
else
baseFile = outputStream.GetInfo(keepReadOpen: false);
// Dispose of the output stream
outputStream.Dispose();
inputStream.Dispose();
}
// Otherwise, just get the info
else
{
baseFile = TryOpenRead(input).GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles);
if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats))
baseFile = AaruFormat.Create(inputStream);
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs))
baseFile = CHDFile.Create(inputStream);
else
baseFile = inputStream.GetInfo(keepReadOpen: false);
}
}
else
{
baseFile = TryOpenRead(input).GetInfo(omitFromScan: omitFromScan, keepReadOpen: false, asFiles: asFiles);
if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFiles.AaruFormats))
baseFile = AaruFormat.Create(inputStream);
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFiles.CHDs))
baseFile = CHDFile.Create(inputStream);
else
baseFile = inputStream.GetInfo(keepReadOpen: false);
}
// Dispose of the input stream
inputStream.Dispose();
// Add unique data from the file
baseFile.RemoveHashes(omitFromScan);
baseFile.Filename = Path.GetFileName(input);
baseFile.Date = (date ? new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss") : string.Empty);

78
SabreTools.Library/IO/StreamExtensions.cs
View File

@@ -45,11 +45,10 @@ namespace SabreTools.Library.IO
/// <param name="size">Size of the input stream</param>
/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated (defaults to none)</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(this Stream input, long size = -1, Hash omitFromScan = 0x0, bool keepReadOpen = false, TreatAsFiles asFiles = 0x00)
public static BaseFile GetInfo(this Stream input, long size = -1, bool keepReadOpen = false)
{
return GetInfoAsync(input, size, omitFromScan, keepReadOpen, asFiles).ConfigureAwait(false).GetAwaiter().GetResult();
return GetInfoAsync(input, size, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult();
}
/// <summary>
@@ -57,53 +56,14 @@ namespace SabreTools.Library.IO
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated (defaults to none)</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, Hash omitFromScan = 0x0, bool keepReadOpen = false, TreatAsFiles asFiles = 0x00)
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, bool keepReadOpen = false)
{
// If we want to automatically set the size
if (size == -1)
size = input.Length;
// We first check to see if it's an AaruFormat if we have to
if (!asFiles.HasFlag(TreatAsFiles.AaruFormats))
{
var aaruFormat = AaruFormat.Create(input);
input.SeekIfPossible();
// If we found a valid AaruFormat
if (aaruFormat != null)
{
if (!keepReadOpen)
input.Dispose();
return aaruFormat;
}
}
// Then, we first check to see if it's a CHD if we have to
if (!asFiles.HasFlag(TreatAsFiles.CHDs))
{
var chd = CHDFile.Create(input);
input.SeekIfPossible();
// If we found a valid CHD
if (chd != null)
{
if (!keepReadOpen)
input.Dispose();
return chd;
}
}
BaseFile rom = new BaseFile()
{
Size = size,
};
try
{
// Get a list of hashers to run over the buffer
@@ -169,28 +129,26 @@ namespace SabreTools.Library.IO
await Task.WhenAll(hashers.Select(h => h.Finalize()));
// Get the results
if (!omitFromScan.HasFlag(Hash.CRC))
rom.CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash();
if (!omitFromScan.HasFlag(Hash.MD5))
rom.MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash();
BaseFile baseFile = new BaseFile()
{
Size = size,
CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash(),
MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash(),
#if NET_FRAMEWORK
if (!omitFromScan.HasFlag(Hash.RIPEMD160))
rom.RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash();
RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash(),
#endif
if (!omitFromScan.HasFlag(Hash.SHA1))
rom.SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash();
if (!omitFromScan.HasFlag(Hash.SHA256))
rom.SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash();
if (!omitFromScan.HasFlag(Hash.SHA384))
rom.SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash();
if (!omitFromScan.HasFlag(Hash.SHA512))
rom.SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash();
if (!omitFromScan.HasFlag(Hash.SpamSum))
rom.SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash();
SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash(),
SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash(),
SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash(),
SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash(),
SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash(),
};
// Dispose of the hashers
loadBuffer.Dispose();
hashers.ForEach(h => h.Dispose());
return baseFile;
}
catch (IOException ex)
{
@@ -204,8 +162,6 @@ namespace SabreTools.Library.IO
else
input.SeekIfPossible();
}
return rom;
}
/// <summary>