mirror of
https://github.com/claunia/SabreTools.git
synced 2025-12-16 19:14:27 +00:00
Extract out IO namespace, Part 2
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
|
||||||
|
using SabreTools.Data;
|
||||||
using SabreTools.Help;
|
using SabreTools.Help;
|
||||||
using SabreTools.IO;
|
using SabreTools.IO;
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Library.DatFiles;
|
||||||
|
|||||||
32
SabreTools.Data/Enums.cs
Normal file
32
SabreTools.Data/Enums.cs
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace SabreTools.Data
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Available hashing types
|
||||||
|
/// </summary>
|
||||||
|
[Flags]
|
||||||
|
public enum Hash
|
||||||
|
{
|
||||||
|
CRC = 1 << 0,
|
||||||
|
MD5 = 1 << 1,
|
||||||
|
#if NET_FRAMEWORK
|
||||||
|
RIPEMD160 = 1 << 2,
|
||||||
|
#endif
|
||||||
|
SHA1 = 1 << 3,
|
||||||
|
SHA256 = 1 << 4,
|
||||||
|
SHA384 = 1 << 5,
|
||||||
|
SHA512 = 1 << 6,
|
||||||
|
SpamSum = 1 << 7,
|
||||||
|
|
||||||
|
// Special combinations
|
||||||
|
Standard = CRC | MD5 | SHA1,
|
||||||
|
#if NET_FRAMEWORK
|
||||||
|
DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum,
|
||||||
|
SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
|
||||||
|
#else
|
||||||
|
DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum,
|
||||||
|
SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -506,7 +506,13 @@ namespace Aaru.Checksums
|
|||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if NET_FRAMEWORK
|
||||||
|
byte[] newString = new byte[count];
|
||||||
|
Array.Copy(cString, newString, count);
|
||||||
|
return newString;
|
||||||
|
#else
|
||||||
return new ReadOnlySpan<byte>(cString, 0, count).ToArray();
|
return new ReadOnlySpan<byte>(cString, 0, count).ToArray();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
@@ -3,10 +3,9 @@ using System.Linq;
|
|||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
using Aaru.Checksums;
|
using Aaru.Checksums;
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.External;
|
|
||||||
|
|
||||||
namespace SabreTools.Library.Tools
|
namespace SabreTools.IO
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Async hashing class wraper
|
/// Async hashing class wraper
|
||||||
@@ -30,7 +29,7 @@ namespace SabreTools.Library.Tools
|
|||||||
switch (HashType)
|
switch (HashType)
|
||||||
{
|
{
|
||||||
case Hash.CRC:
|
case Hash.CRC:
|
||||||
_hasher = new OptimizedCRC();
|
_hasher = new OptimizedCRC.OptimizedCRC();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Hash.MD5:
|
case Hash.MD5:
|
||||||
@@ -78,7 +77,7 @@ namespace SabreTools.Library.Tools
|
|||||||
switch (HashType)
|
switch (HashType)
|
||||||
{
|
{
|
||||||
case Hash.CRC:
|
case Hash.CRC:
|
||||||
(_hasher as OptimizedCRC).Update(buffer, 0, size);
|
(_hasher as OptimizedCRC.OptimizedCRC).Update(buffer, 0, size);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Hash.MD5:
|
case Hash.MD5:
|
||||||
@@ -107,7 +106,7 @@ namespace SabreTools.Library.Tools
|
|||||||
switch (HashType)
|
switch (HashType)
|
||||||
{
|
{
|
||||||
case Hash.CRC:
|
case Hash.CRC:
|
||||||
(_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0);
|
(_hasher as OptimizedCRC.OptimizedCRC).Update(emptyBuffer, 0, 0);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case Hash.MD5:
|
case Hash.MD5:
|
||||||
@@ -135,7 +134,7 @@ namespace SabreTools.Library.Tools
|
|||||||
switch (HashType)
|
switch (HashType)
|
||||||
{
|
{
|
||||||
case Hash.CRC:
|
case Hash.CRC:
|
||||||
return BitConverter.GetBytes((_hasher as OptimizedCRC).Value).Reverse().ToArray();
|
return BitConverter.GetBytes((_hasher as OptimizedCRC.OptimizedCRC).Value).Reverse().ToArray();
|
||||||
|
|
||||||
case Hash.MD5:
|
case Hash.MD5:
|
||||||
#if NET_FRAMEWORK
|
#if NET_FRAMEWORK
|
||||||
@@ -24,9 +24,9 @@
|
|||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace SabreTools.Library.External
|
namespace OptimizedCRC
|
||||||
{
|
{
|
||||||
public class OptimizedCRC : IDisposable
|
internal class OptimizedCRC : IDisposable
|
||||||
{
|
{
|
||||||
private const uint kCrcPoly = 0xEDB88320;
|
private const uint kCrcPoly = 0xEDB88320;
|
||||||
private const uint kInitial = 0xFFFFFFFF;
|
private const uint kInitial = 0xFFFFFFFF;
|
||||||
66
SabreTools.IO/StreamExtensions.cs
Normal file
66
SabreTools.IO/StreamExtensions.cs
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
using SabreTools.Logging;
|
||||||
|
|
||||||
|
namespace SabreTools.IO
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Extensions to Stream functionality
|
||||||
|
/// </summary>
|
||||||
|
public static class StreamExtensions
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Add an aribtrary number of bytes to the inputted stream
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">Stream to be appended to</param>
|
||||||
|
/// <param name="output">Outputted stream</param>
|
||||||
|
/// <param name="bytesToAddToHead">Bytes to be added to head of stream</param>
|
||||||
|
/// <param name="bytesToAddToTail">Bytes to be added to tail of stream</param>
|
||||||
|
public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail)
|
||||||
|
{
|
||||||
|
// Write out prepended bytes
|
||||||
|
if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0)
|
||||||
|
output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length);
|
||||||
|
|
||||||
|
// Now copy the existing file over
|
||||||
|
input.CopyTo(output);
|
||||||
|
|
||||||
|
// Write out appended bytes
|
||||||
|
if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0)
|
||||||
|
output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Seek to a specific point in the stream, if possible
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">Input stream to try seeking on</param>
|
||||||
|
/// <param name="offset">Optional offset to seek to</param>
|
||||||
|
public static long SeekIfPossible(this Stream input, long offset = 0)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (input.CanSeek)
|
||||||
|
{
|
||||||
|
if (offset < 0)
|
||||||
|
return input.Seek(offset, SeekOrigin.End);
|
||||||
|
else if (offset >= 0)
|
||||||
|
return input.Seek(offset, SeekOrigin.Begin);
|
||||||
|
}
|
||||||
|
|
||||||
|
return input.Position;
|
||||||
|
}
|
||||||
|
catch (NotSupportedException ex)
|
||||||
|
{
|
||||||
|
LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed");
|
||||||
|
}
|
||||||
|
catch (NotImplementedException ex)
|
||||||
|
{
|
||||||
|
LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed");
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2779,7 +2779,7 @@ namespace SabreTools.Library.DatFiles
|
|||||||
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
|
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
|
||||||
{
|
{
|
||||||
// Get the file informations that we will be using
|
// Get the file informations that we will be using
|
||||||
Rom headerless = new Rom(transformStream.GetInfo(keepReadOpen: true));
|
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
|
||||||
|
|
||||||
// If we have duplicates and we're not filtering
|
// If we have duplicates and we're not filtering
|
||||||
if (ShouldRebuild(headerless, transformStream, false, out dupes))
|
if (ShouldRebuild(headerless, transformStream, false, out dupes))
|
||||||
@@ -2851,7 +2851,7 @@ namespace SabreTools.Library.DatFiles
|
|||||||
string machinename = null;
|
string machinename = null;
|
||||||
|
|
||||||
// Get the item from the current file
|
// Get the item from the current file
|
||||||
Rom item = new Rom(stream.GetInfo(keepReadOpen: true));
|
Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true));
|
||||||
item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name);
|
item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name);
|
||||||
item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name);
|
item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name);
|
||||||
|
|
||||||
|
|||||||
@@ -181,34 +181,6 @@ namespace SabreTools.Library.DatFiles
|
|||||||
SHA512,
|
SHA512,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Available hashing types
|
|
||||||
/// </summary>
|
|
||||||
[Flags]
|
|
||||||
public enum Hash
|
|
||||||
{
|
|
||||||
CRC = 1 << 0,
|
|
||||||
MD5 = 1 << 1,
|
|
||||||
#if NET_FRAMEWORK
|
|
||||||
RIPEMD160 = 1 << 2,
|
|
||||||
#endif
|
|
||||||
SHA1 = 1 << 3,
|
|
||||||
SHA256 = 1 << 4,
|
|
||||||
SHA384 = 1 << 5,
|
|
||||||
SHA512 = 1 << 6,
|
|
||||||
SpamSum = 1 << 7,
|
|
||||||
|
|
||||||
// Special combinations
|
|
||||||
Standard = CRC | MD5 | SHA1,
|
|
||||||
#if NET_FRAMEWORK
|
|
||||||
DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum,
|
|
||||||
SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
|
|
||||||
#else
|
|
||||||
DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum,
|
|
||||||
SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines merging tag handling for DAT output
|
/// Determines merging tag handling for DAT output
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
|
using SabreTools.Data;
|
||||||
using SabreTools.IO;
|
using SabreTools.IO;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
|
using SabreTools.IO;
|
||||||
using SabreTools.Data;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.FileTypes.Aaru;
|
using SabreTools.Library.FileTypes.Aaru;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
using System.IO;
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
|
using SabreTools.IO;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
|
using SabreTools.Logging;
|
||||||
|
using Compress.ThreadReaders;
|
||||||
|
|
||||||
namespace SabreTools.Library.FileTypes
|
namespace SabreTools.Library.FileTypes
|
||||||
{
|
{
|
||||||
@@ -135,7 +141,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
|
|
||||||
if (getHashes)
|
if (getHashes)
|
||||||
{
|
{
|
||||||
BaseFile temp = stream.GetInfo(hashes: this.AvailableHashes);
|
BaseFile temp = GetInfo(stream, hashes: this.AvailableHashes);
|
||||||
if (temp != null)
|
if (temp != null)
|
||||||
{
|
{
|
||||||
this.Parent = temp.Parent;
|
this.Parent = temp.Parent;
|
||||||
@@ -156,5 +162,131 @@ namespace SabreTools.Library.FileTypes
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
#region Static Methods
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Retrieve file information for a single file
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">Filename to get information from</param>
|
||||||
|
/// <param name="size">Size of the input stream</param>
|
||||||
|
/// <param name="hashes">Hashes to include in the information</param>
|
||||||
|
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
|
||||||
|
/// <returns>Populated BaseFile object if success, empty one on error</returns>
|
||||||
|
public static BaseFile GetInfo(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
|
||||||
|
{
|
||||||
|
// If we want to automatically set the size
|
||||||
|
if (size == -1)
|
||||||
|
size = input.Length;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Get a list of hashers to run over the buffer
|
||||||
|
List<Hasher> hashers = new List<Hasher>();
|
||||||
|
|
||||||
|
if (hashes.HasFlag(Hash.CRC))
|
||||||
|
hashers.Add(new Hasher(Hash.CRC));
|
||||||
|
if (hashes.HasFlag(Hash.MD5))
|
||||||
|
hashers.Add(new Hasher(Hash.MD5));
|
||||||
|
#if NET_FRAMEWORK
|
||||||
|
if (hashes.HasFlag(Hash.RIPEMD160))
|
||||||
|
hashers.Add(new Hasher(Hash.RIPEMD160));
|
||||||
|
#endif
|
||||||
|
if (hashes.HasFlag(Hash.SHA1))
|
||||||
|
hashers.Add(new Hasher(Hash.SHA1));
|
||||||
|
if (hashes.HasFlag(Hash.SHA256))
|
||||||
|
hashers.Add(new Hasher(Hash.SHA256));
|
||||||
|
if (hashes.HasFlag(Hash.SHA384))
|
||||||
|
hashers.Add(new Hasher(Hash.SHA384));
|
||||||
|
if (hashes.HasFlag(Hash.SHA512))
|
||||||
|
hashers.Add(new Hasher(Hash.SHA512));
|
||||||
|
if (hashes.HasFlag(Hash.SpamSum))
|
||||||
|
hashers.Add(new Hasher(Hash.SpamSum));
|
||||||
|
|
||||||
|
// Initialize the hashing helpers
|
||||||
|
var loadBuffer = new ThreadLoadBuffer(input);
|
||||||
|
int buffersize = 3 * 1024 * 1024;
|
||||||
|
byte[] buffer0 = new byte[buffersize];
|
||||||
|
byte[] buffer1 = new byte[buffersize];
|
||||||
|
|
||||||
|
/*
|
||||||
|
Please note that some of the following code is adapted from
|
||||||
|
RomVault. This is a modified version of how RomVault does
|
||||||
|
threaded hashing. As such, some of the terminology and code
|
||||||
|
is the same, though variable names and comments may have
|
||||||
|
been tweaked to better fit this code base.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Pre load the first buffer
|
||||||
|
long refsize = size;
|
||||||
|
int next = refsize > buffersize ? buffersize : (int)refsize;
|
||||||
|
input.Read(buffer0, 0, next);
|
||||||
|
int current = next;
|
||||||
|
refsize -= next;
|
||||||
|
bool bufferSelect = true;
|
||||||
|
|
||||||
|
while (current > 0)
|
||||||
|
{
|
||||||
|
// Trigger the buffer load on the second buffer
|
||||||
|
next = refsize > buffersize ? buffersize : (int)refsize;
|
||||||
|
if (next > 0)
|
||||||
|
loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next);
|
||||||
|
|
||||||
|
byte[] buffer = bufferSelect ? buffer0 : buffer1;
|
||||||
|
|
||||||
|
// Run hashes in parallel
|
||||||
|
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current));
|
||||||
|
|
||||||
|
// Wait for the load buffer worker, if needed
|
||||||
|
if (next > 0)
|
||||||
|
loadBuffer.Wait();
|
||||||
|
|
||||||
|
// Setup for the next hashing step
|
||||||
|
current = next;
|
||||||
|
refsize -= next;
|
||||||
|
bufferSelect = !bufferSelect;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finalize all hashing helpers
|
||||||
|
loadBuffer.Finish();
|
||||||
|
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize());
|
||||||
|
|
||||||
|
// Get the results
|
||||||
|
BaseFile baseFile = new BaseFile()
|
||||||
|
{
|
||||||
|
Size = size,
|
||||||
|
CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null,
|
||||||
|
MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null,
|
||||||
|
#if NET_FRAMEWORK
|
||||||
|
RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null,
|
||||||
|
#endif
|
||||||
|
SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null,
|
||||||
|
SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null,
|
||||||
|
SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null,
|
||||||
|
SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null,
|
||||||
|
SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Dispose of the hashers
|
||||||
|
loadBuffer.Dispose();
|
||||||
|
hashers.ForEach(h => h.Dispose());
|
||||||
|
|
||||||
|
return baseFile;
|
||||||
|
}
|
||||||
|
catch (IOException ex)
|
||||||
|
{
|
||||||
|
LoggerImpl.Warning(ex, "An exception occurred during hashing.");
|
||||||
|
return new BaseFile();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (!keepReadOpen)
|
||||||
|
input.Dispose();
|
||||||
|
else
|
||||||
|
input.SeekIfPossible();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -239,7 +239,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
var gz = new gZip();
|
var gz = new gZip();
|
||||||
ZipReturn ret = gz.ZipFileOpen(this.Filename);
|
ZipReturn ret = gz.ZipFileOpen(this.Filename);
|
||||||
ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
|
ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
|
||||||
gzipEntryRom = gzstream.GetInfo(hashes: this.AvailableHashes);
|
gzipEntryRom = GetInfo(gzstream, hashes: this.AvailableHashes);
|
||||||
gzipEntryRom.Filename = gz.Filename(0);
|
gzipEntryRom.Filename = gz.Filename(0);
|
||||||
gzipEntryRom.Parent = gamename;
|
gzipEntryRom.Parent = gamename;
|
||||||
gzipEntryRom.Date = (gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);
|
gzipEntryRom.Date = (gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);
|
||||||
@@ -461,7 +461,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
outDir = Path.GetFullPath(outDir);
|
outDir = Path.GetFullPath(outDir);
|
||||||
|
|
||||||
// Now get the Rom info for the file so we have hashes and size
|
// Now get the Rom info for the file so we have hashes and size
|
||||||
rom = new Rom(inputStream.GetInfo(keepReadOpen: true));
|
rom = new Rom(GetInfo(inputStream, keepReadOpen: true));
|
||||||
|
|
||||||
// Get the output file name
|
// Get the output file name
|
||||||
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));
|
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
using SharpCompress.Archives;
|
using SharpCompress.Archives;
|
||||||
@@ -199,7 +199,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
{
|
{
|
||||||
using (Stream entryStream = entry.OpenEntryStream())
|
using (Stream entryStream = entry.OpenEntryStream())
|
||||||
{
|
{
|
||||||
rarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
|
rarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
using SabreTools.Library.Tools;
|
using SabreTools.Library.Tools;
|
||||||
@@ -307,7 +307,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
// Otherwise, use the stream directly
|
// Otherwise, use the stream directly
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
|
zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fill in comon details and add to the list
|
// Fill in comon details and add to the list
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
using SabreTools.Library.Tools;
|
using SabreTools.Library.Tools;
|
||||||
@@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
{
|
{
|
||||||
using (Stream entryStream = entry.OpenEntryStream())
|
using (Stream entryStream = entry.OpenEntryStream())
|
||||||
{
|
{
|
||||||
tarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
|
tarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
var xzStream = new XZStream(File.OpenRead(this.Filename));
|
var xzStream = new XZStream(File.OpenRead(this.Filename));
|
||||||
xzEntryRom = xzStream.GetInfo(hashes: this.AvailableHashes);
|
xzEntryRom = GetInfo(xzStream, hashes: this.AvailableHashes);
|
||||||
xzEntryRom.Filename = gamename;
|
xzEntryRom.Filename = gamename;
|
||||||
xzStream.Dispose();
|
xzStream.Dispose();
|
||||||
}
|
}
|
||||||
@@ -360,7 +360,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
outDir = Path.GetFullPath(outDir);
|
outDir = Path.GetFullPath(outDir);
|
||||||
|
|
||||||
// Now get the Rom info for the file so we have hashes and size
|
// Now get the Rom info for the file so we have hashes and size
|
||||||
rom = new Rom(inputStream.GetInfo(keepReadOpen: true));
|
rom = new Rom(GetInfo(inputStream, keepReadOpen: true));
|
||||||
|
|
||||||
// Get the output file name
|
// Get the output file name
|
||||||
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));
|
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.IO;
|
using SabreTools.Library.IO;
|
||||||
using SabreTools.Library.Tools;
|
using SabreTools.Library.Tools;
|
||||||
@@ -308,7 +308,7 @@ namespace SabreTools.Library.FileTypes
|
|||||||
// Otherwise, use the stream directly
|
// Otherwise, use the stream directly
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
|
zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fill in comon details and add to the list
|
// Fill in comon details and add to the list
|
||||||
|
|||||||
@@ -391,7 +391,7 @@ namespace SabreTools.Library.IO
|
|||||||
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
|
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
|
||||||
baseFile = CHDFile.Create(inputStream);
|
baseFile = CHDFile.Create(inputStream);
|
||||||
else
|
else
|
||||||
baseFile = inputStream.GetInfo(hashes: hashes, keepReadOpen: false);
|
baseFile = BaseFile.GetInfo(inputStream, hashes: hashes, keepReadOpen: false);
|
||||||
|
|
||||||
// Dispose of the input stream
|
// Dispose of the input stream
|
||||||
inputStream?.Dispose();
|
inputStream?.Dispose();
|
||||||
|
|||||||
@@ -1,208 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
using SabreTools.Data;
|
|
||||||
using SabreTools.Logging;
|
|
||||||
using SabreTools.Library.DatFiles;
|
|
||||||
using SabreTools.Library.FileTypes;
|
|
||||||
using SabreTools.Library.Tools;
|
|
||||||
using Compress.ThreadReaders;
|
|
||||||
|
|
||||||
namespace SabreTools.Library.IO
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Extensions to Stream functionality
|
|
||||||
/// </summary>
|
|
||||||
public static class StreamExtensions
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Add an aribtrary number of bytes to the inputted stream
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Stream to be appended to</param>
|
|
||||||
/// <param name="output">Outputted stream</param>
|
|
||||||
/// <param name="bytesToAddToHead">Bytes to be added to head of stream</param>
|
|
||||||
/// <param name="bytesToAddToTail">Bytes to be added to tail of stream</param>
|
|
||||||
public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail)
|
|
||||||
{
|
|
||||||
// Write out prepended bytes
|
|
||||||
if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0)
|
|
||||||
output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length);
|
|
||||||
|
|
||||||
// Now copy the existing file over
|
|
||||||
input.CopyTo(output);
|
|
||||||
|
|
||||||
// Write out appended bytes
|
|
||||||
if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0)
|
|
||||||
output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Retrieve file information for a single file
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Filename to get information from</param>
|
|
||||||
/// <param name="size">Size of the input stream</param>
|
|
||||||
/// <param name="hashes">Hashes to include in the information</param>
|
|
||||||
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
|
|
||||||
/// <returns>Populated BaseFile object if success, empty one on error</returns>
|
|
||||||
public static BaseFile GetInfo(this Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
|
|
||||||
{
|
|
||||||
return GetInfoAsync(input, size, hashes, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Retrieve file information for a single file
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Filename to get information from</param>
|
|
||||||
/// <param name="size">Size of the input stream</param>
|
|
||||||
/// <param name="hashes">Hashes to include in the information</param>
|
|
||||||
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
|
|
||||||
/// <returns>Populated BaseFile object if success, empty one on error</returns>
|
|
||||||
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
|
|
||||||
{
|
|
||||||
// If we want to automatically set the size
|
|
||||||
if (size == -1)
|
|
||||||
size = input.Length;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
// Get a list of hashers to run over the buffer
|
|
||||||
List<Hasher> hashers = new List<Hasher>();
|
|
||||||
|
|
||||||
if (hashes.HasFlag(Hash.CRC))
|
|
||||||
hashers.Add(new Hasher(Hash.CRC));
|
|
||||||
if (hashes.HasFlag(Hash.MD5))
|
|
||||||
hashers.Add(new Hasher(Hash.MD5));
|
|
||||||
#if NET_FRAMEWORK
|
|
||||||
if (hashes.HasFlag(Hash.RIPEMD160))
|
|
||||||
hashers.Add(new Hasher(Hash.RIPEMD160));
|
|
||||||
#endif
|
|
||||||
if (hashes.HasFlag(Hash.SHA1))
|
|
||||||
hashers.Add(new Hasher(Hash.SHA1));
|
|
||||||
if (hashes.HasFlag(Hash.SHA256))
|
|
||||||
hashers.Add(new Hasher(Hash.SHA256));
|
|
||||||
if (hashes.HasFlag(Hash.SHA384))
|
|
||||||
hashers.Add(new Hasher(Hash.SHA384));
|
|
||||||
if (hashes.HasFlag(Hash.SHA512))
|
|
||||||
hashers.Add(new Hasher(Hash.SHA512));
|
|
||||||
if (hashes.HasFlag(Hash.SpamSum))
|
|
||||||
hashers.Add(new Hasher(Hash.SpamSum));
|
|
||||||
|
|
||||||
// Initialize the hashing helpers
|
|
||||||
var loadBuffer = new ThreadLoadBuffer(input);
|
|
||||||
int buffersize = 3 * 1024 * 1024;
|
|
||||||
byte[] buffer0 = new byte[buffersize];
|
|
||||||
byte[] buffer1 = new byte[buffersize];
|
|
||||||
|
|
||||||
/*
|
|
||||||
Please note that some of the following code is adapted from
|
|
||||||
RomVault. This is a modified version of how RomVault does
|
|
||||||
threaded hashing. As such, some of the terminology and code
|
|
||||||
is the same, though variable names and comments may have
|
|
||||||
been tweaked to better fit this code base.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Pre load the first buffer
|
|
||||||
long refsize = size;
|
|
||||||
int next = refsize > buffersize ? buffersize : (int)refsize;
|
|
||||||
input.Read(buffer0, 0, next);
|
|
||||||
int current = next;
|
|
||||||
refsize -= next;
|
|
||||||
bool bufferSelect = true;
|
|
||||||
|
|
||||||
while (current > 0)
|
|
||||||
{
|
|
||||||
// Trigger the buffer load on the second buffer
|
|
||||||
next = refsize > buffersize ? buffersize : (int)refsize;
|
|
||||||
if (next > 0)
|
|
||||||
loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next);
|
|
||||||
|
|
||||||
byte[] buffer = bufferSelect ? buffer0 : buffer1;
|
|
||||||
|
|
||||||
// Run hashes in parallel
|
|
||||||
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current));
|
|
||||||
|
|
||||||
// Wait for the load buffer worker, if needed
|
|
||||||
if (next > 0)
|
|
||||||
loadBuffer.Wait();
|
|
||||||
|
|
||||||
// Setup for the next hashing step
|
|
||||||
current = next;
|
|
||||||
refsize -= next;
|
|
||||||
bufferSelect = !bufferSelect;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Finalize all hashing helpers
|
|
||||||
loadBuffer.Finish();
|
|
||||||
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize());
|
|
||||||
|
|
||||||
// Get the results
|
|
||||||
BaseFile baseFile = new BaseFile()
|
|
||||||
{
|
|
||||||
Size = size,
|
|
||||||
CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null,
|
|
||||||
MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null,
|
|
||||||
#if NET_FRAMEWORK
|
|
||||||
RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null,
|
|
||||||
#endif
|
|
||||||
SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null,
|
|
||||||
SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null,
|
|
||||||
SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null,
|
|
||||||
SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null,
|
|
||||||
SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Dispose of the hashers
|
|
||||||
loadBuffer.Dispose();
|
|
||||||
hashers.ForEach(h => h.Dispose());
|
|
||||||
|
|
||||||
return baseFile;
|
|
||||||
}
|
|
||||||
catch (IOException ex)
|
|
||||||
{
|
|
||||||
LoggerImpl.Warning(ex, "An exception occurred during hashing.");
|
|
||||||
return new BaseFile();
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
if (!keepReadOpen)
|
|
||||||
input.Dispose();
|
|
||||||
else
|
|
||||||
input.SeekIfPossible();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Seek to a specific point in the stream, if possible
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Input stream to try seeking on</param>
|
|
||||||
/// <param name="offset">Optional offset to seek to</param>
|
|
||||||
public static long SeekIfPossible(this Stream input, long offset = 0)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (input.CanSeek)
|
|
||||||
{
|
|
||||||
if (offset < 0)
|
|
||||||
return input.Seek(offset, SeekOrigin.End);
|
|
||||||
else if (offset >= 0)
|
|
||||||
return input.Seek(offset, SeekOrigin.Begin);
|
|
||||||
}
|
|
||||||
|
|
||||||
return input.Position;
|
|
||||||
}
|
|
||||||
catch (NotSupportedException ex)
|
|
||||||
{
|
|
||||||
LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed");
|
|
||||||
}
|
|
||||||
catch (NotImplementedException ex)
|
|
||||||
{
|
|
||||||
LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed");
|
|
||||||
}
|
|
||||||
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
using SabreTools.Data;
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Library.DatFiles;
|
||||||
using SabreTools.Library.DatItems;
|
using SabreTools.Library.DatItems;
|
||||||
using SabreTools.Library.FileTypes;
|
using SabreTools.Library.FileTypes;
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using System.IO;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
using SabreTools.Data;
|
||||||
using SabreTools.Help;
|
using SabreTools.Help;
|
||||||
using SabreTools.IO;
|
using SabreTools.IO;
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Library.DatFiles;
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using SabreTools.Data;
|
||||||
using SabreTools.Help;
|
using SabreTools.Help;
|
||||||
using SabreTools.IO;
|
using SabreTools.IO;
|
||||||
using SabreTools.Library.DatFiles;
|
using SabreTools.Library.DatFiles;
|
||||||
|
|||||||
Reference in New Issue
Block a user