Extract out IO namespace, Part 2

This commit is contained in:
Matt Nadareski
2020-12-07 22:32:37 -08:00
parent 96e2afcfe4
commit 53f5d07ae4
23 changed files with 268 additions and 263 deletions

View File

@@ -2779,7 +2779,7 @@ namespace SabreTools.Library.DatFiles
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(transformStream.GetInfo(keepReadOpen: true));
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(headerless, transformStream, false, out dupes))
@@ -2851,7 +2851,7 @@ namespace SabreTools.Library.DatFiles
string machinename = null;
// Get the item from the current file
Rom item = new Rom(stream.GetInfo(keepReadOpen: true));
Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true));
item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name);
item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name);

View File

@@ -181,34 +181,6 @@ namespace SabreTools.Library.DatFiles
SHA512,
}
/// <summary>
/// Available hashing types
/// </summary>
[Flags]
public enum Hash
{
CRC = 1 << 0,
MD5 = 1 << 1,
#if NET_FRAMEWORK
RIPEMD160 = 1 << 2,
#endif
SHA1 = 1 << 3,
SHA256 = 1 << 4,
SHA384 = 1 << 5,
SHA512 = 1 << 6,
SpamSum = 1 << 7,
// Special combinations
Standard = CRC | MD5 | SHA1,
#if NET_FRAMEWORK
DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum,
SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
#else
DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum,
SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
#endif
}
/// <summary>
/// Determines merging tag handling for DAT output
/// </summary>

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Text;
using SabreTools.Data;
using SabreTools.IO;
using SabreTools.Library.DatItems;
using SabreTools.Library.IO;

View File

@@ -1,58 +0,0 @@
// /***************************************************************************
// Aaru Data Preservation Suite
// ----------------------------------------------------------------------------
//
// Filename : IChecksum.cs
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// Component : Checksums.
//
// --[ Description ] ----------------------------------------------------------
//
// Provides an interface for implementing checksums and hashes.
//
// --[ License ] --------------------------------------------------------------
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2020 Natalia Portillo
// ****************************************************************************/
namespace Aaru.CommonTypes.Interfaces
{
public interface IChecksum
{
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of buffer to hash.</param>
void Update(byte[] data, uint len);
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
void Update(byte[] data);
/// <summary>Returns a byte array of the hash value.</summary>
byte[] Final();
/// <summary>Returns a hexadecimal representation of the hash value.</summary>
string End();
}
}

View File

@@ -1,555 +0,0 @@
// /***************************************************************************
// Aaru Data Preservation Suite
// ----------------------------------------------------------------------------
//
// Filename : SpamSumContext.cs
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// Component : Checksums.
//
// --[ Description ] ----------------------------------------------------------
//
// Implements the SpamSum fuzzy hashing algorithm.
//
// --[ License ] --------------------------------------------------------------
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2020 Natalia Portillo
// ****************************************************************************/
// Based on ssdeep
// Copyright (C) 2002 Andrew Tridgell <tridge@samba.org>
// Copyright (C) 2006 ManTech International Corporation
// Copyright (C) 2013 Helmut Grohne <helmut@subdivi.de>
//
// Earlier versions of this code were named fuzzy.c and can be found at:
// http://www.samba.org/ftp/unpacked/junkcode/spamsum/
// http://ssdeep.sf.net/
using System;
using System.Runtime.CompilerServices;
using System.Text;
using Aaru.CommonTypes.Interfaces;
namespace Aaru.Checksums
{
/// <summary>Implements the SpamSum fuzzy hashing algorithm.</summary>
public sealed class SpamSumContext : IChecksum, IDisposable
{
const uint ROLLING_WINDOW = 7;
const uint MIN_BLOCKSIZE = 3;
const uint HASH_PRIME = 0x01000193;
const uint HASH_INIT = 0x28021967;
const uint NUM_BLOCKHASHES = 31;
const uint SPAMSUM_LENGTH = 64;
const uint FUZZY_MAX_RESULT = (2 * SPAMSUM_LENGTH) + 20;
//"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
readonly byte[] _b64 =
{
0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52,
0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A,
0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x30, 0x31,
0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2F
};
FuzzyState _self;
/// <summary>Initializes the SpamSum structures</summary>
public SpamSumContext()
{
_self = new FuzzyState
{
Bh = new BlockhashContext[NUM_BLOCKHASHES]
};
for(int i = 0; i < NUM_BLOCKHASHES; i++)
_self.Bh[i].Digest = new byte[SPAMSUM_LENGTH];
_self.Bhstart = 0;
_self.Bhend = 1;
_self.Bh[0].H = HASH_INIT;
_self.Bh[0].Halfh = HASH_INIT;
_self.Bh[0].Digest[0] = 0;
_self.Bh[0].Halfdigest = 0;
_self.Bh[0].Dlen = 0;
_self.TotalSize = 0;
roll_init();
}
/// <inheritdoc />
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of buffer to hash.</param>
public void Update(byte[] data, uint len)
{
_self.TotalSize += len;
for(int i = 0; i < len; i++)
fuzzy_engine_step(data[i]);
}
/// <inheritdoc />
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
public void Update(byte[] data) => Update(data, (uint)data.Length);
/// <inheritdoc />
/// <summary>Returns a byte array of the hash value.</summary>
public byte[] Final()
{
FuzzyDigest(out byte[] result);
return CToArray(result);
}
/// <inheritdoc />
/// <summary>Returns a base64 representation of the hash value.</summary>
public string End()
{
FuzzyDigest(out byte[] result);
return CToString(result);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void roll_init() => _self.Roll = new RollState
{
Window = new byte[ROLLING_WINDOW]
};
/*
* a rolling hash, based on the Adler checksum. By using a rolling hash
* we can perform auto resynchronisation after inserts/deletes
* internally, h1 is the sum of the bytes in the window and h2
* is the sum of the bytes times the index
* h3 is a shift/xor based rolling hash, and is mostly needed to ensure that
* we can cope with large blocksize values
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void roll_hash(byte c)
{
_self.Roll.H2 -= _self.Roll.H1;
_self.Roll.H2 += ROLLING_WINDOW * c;
_self.Roll.H1 += c;
_self.Roll.H1 -= _self.Roll.Window[_self.Roll.N % ROLLING_WINDOW];
_self.Roll.Window[_self.Roll.N % ROLLING_WINDOW] = c;
_self.Roll.N++;
/* The original spamsum AND'ed this value with 0xFFFFFFFF which
* in theory should have no effect. This AND has been removed
* for performance (jk) */
_self.Roll.H3 <<= 5;
_self.Roll.H3 ^= c;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
uint roll_sum() => _self.Roll.H1 + _self.Roll.H2 + _self.Roll.H3;
/* A simple non-rolling hash, based on the FNV hash. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static uint sum_hash(byte c, uint h) => (h * HASH_PRIME) ^ c;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static uint SSDEEP_BS(uint index) => MIN_BLOCKSIZE << (int)index;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_try_fork_blockhash()
{
if(_self.Bhend >= NUM_BLOCKHASHES)
return;
if(_self.Bhend == 0) // assert
throw new Exception("Assertion failed");
uint obh = _self.Bhend - 1;
uint nbh = _self.Bhend;
_self.Bh[nbh].H = _self.Bh[obh].H;
_self.Bh[nbh].Halfh = _self.Bh[obh].Halfh;
_self.Bh[nbh].Digest[0] = 0;
_self.Bh[nbh].Halfdigest = 0;
_self.Bh[nbh].Dlen = 0;
++_self.Bhend;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_try_reduce_blockhash()
{
if(_self.Bhstart >= _self.Bhend)
throw new Exception("Assertion failed");
if(_self.Bhend - _self.Bhstart < 2)
/* Need at least two working hashes. */
return;
if((ulong)SSDEEP_BS(_self.Bhstart) * SPAMSUM_LENGTH >= _self.TotalSize)
/* Initial blocksize estimate would select this or a smaller
* blocksize. */
return;
if(_self.Bh[_self.Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2)
/* Estimate adjustment would select this blocksize. */
return;
/* At this point we are clearly no longer interested in the
* start_blocksize. Get rid of it. */
++_self.Bhstart;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_engine_step(byte c)
{
uint i;
/* At each character we update the rolling hash and the normal hashes.
* When the rolling hash hits a reset value then we emit a normal hash
* as a element of the signature and reset the normal hash. */
roll_hash(c);
ulong h = roll_sum();
for(i = _self.Bhstart; i < _self.Bhend; ++i)
{
_self.Bh[i].H = sum_hash(c, _self.Bh[i].H);
_self.Bh[i].Halfh = sum_hash(c, _self.Bh[i].Halfh);
}
for(i = _self.Bhstart; i < _self.Bhend; ++i)
{
/* With growing blocksize almost no runs fail the next test. */
if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
/* Once this condition is false for one bs, it is
* automatically false for all further bs. I.e. if
* h === -1 (mod 2*bs) then h === -1 (mod bs). */
break;
/* We have hit a reset point. We now emit hashes which are
* based on all characters in the piece of the message between
* the last reset point and this one */
if(0 == _self.Bh[i].Dlen)
fuzzy_try_fork_blockhash();
_self.Bh[i].Digest[_self.Bh[i].Dlen] = _b64[_self.Bh[i].H % 64];
_self.Bh[i].Halfdigest = _b64[_self.Bh[i].Halfh % 64];
if(_self.Bh[i].Dlen < SPAMSUM_LENGTH - 1)
{
/* We can have a problem with the tail overflowing. The
* easiest way to cope with this is to only reset the
* normal hash if we have room for more characters in
* our signature. This has the effect of combining the
* last few pieces of the message into a single piece
* */
_self.Bh[i].Digest[++_self.Bh[i].Dlen] = 0;
_self.Bh[i].H = HASH_INIT;
if(_self.Bh[i].Dlen >= SPAMSUM_LENGTH / 2)
continue;
_self.Bh[i].Halfh = HASH_INIT;
_self.Bh[i].Halfdigest = 0;
}
else
fuzzy_try_reduce_blockhash();
}
}
// CLAUNIA: Flags seems to never be used in ssdeep, so I just removed it for code simplicity
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void FuzzyDigest(out byte[] result)
{
var sb = new StringBuilder();
uint bi = _self.Bhstart;
uint h = roll_sum();
int remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */
result = new byte[FUZZY_MAX_RESULT];
/* Verify that our elimination was not overeager. */
if(!(bi == 0 || ((ulong)SSDEEP_BS(bi) / 2) * SPAMSUM_LENGTH < _self.TotalSize))
throw new Exception("Assertion failed");
int resultOff = 0;
/* Initial blocksize guess. */
while((ulong)SSDEEP_BS(bi) * SPAMSUM_LENGTH < _self.TotalSize)
{
++bi;
if(bi >= NUM_BLOCKHASHES)
throw new OverflowException("The input exceeds data types.");
}
/* Adapt blocksize guess to actual digest length. */
while(bi >= _self.Bhend)
--bi;
while(bi > _self.Bhstart &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
--bi;
if(bi > 0 &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
throw new Exception("Assertion failed");
sb.AppendFormat("{0}:", SSDEEP_BS(bi));
int i = Encoding.ASCII.GetBytes(sb.ToString()).Length;
if(i <= 0)
/* Maybe snprintf has set errno here? */
throw new OverflowException("The input exceeds data types.");
if(i >= remain)
throw new Exception("Assertion failed");
remain -= i;
Array.Copy(Encoding.ASCII.GetBytes(sb.ToString()), 0, result, 0, i);
resultOff = i;
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
if(h != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _b64[_self.Bh[bi].H % 64];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
else if(_self.Bh[bi].Digest[i] != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _self.Bh[bi].Digest[i];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = 0x3A; // ':'
--remain;
if(bi < _self.Bhend - 1)
{
++bi;
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
if(h != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
h = _self.Bh[bi].Halfh;
result[resultOff] = _b64[h % 64];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
else
{
i = _self.Bh[bi].Halfdigest;
if(i != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = (byte)i;
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
}
}
else if(h != 0)
{
if(_self.Bh[bi].Dlen != 0)
throw new Exception("Assertion failed");
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = _b64[_self.Bh[bi].H % 64];
/* No need to bother with FUZZY_FLAG_ELIMSEQ, because this
* digest has length 1. */
--remain;
}
result[resultOff] = 0;
}
/// <summary>Gets the hash of a file</summary>
/// <param name="filename">File path.</param>
public static byte[] File(string filename) =>
throw new NotImplementedException("SpamSum does not have a binary representation.");
/// <summary>Gets the hash of a file in hexadecimal and as a byte array.</summary>
/// <param name="filename">File path.</param>
/// <param name="hash">Byte array of the hash value.</param>
public static string File(string filename, out byte[] hash) =>
throw new NotImplementedException("Not yet implemented.");
/// <summary>Gets the hash of the specified data buffer.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of the data buffer to hash.</param>
/// <param name="hash">null</param>
/// <returns>Base64 representation of SpamSum $blocksize:$hash:$hash</returns>
public static string Data(byte[] data, uint len, out byte[] hash)
{
var fuzzyContext = new SpamSumContext();
fuzzyContext.Update(data, len);
hash = null;
return fuzzyContext.End();
}
/// <summary>Gets the hash of the specified data buffer.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="hash">null</param>
/// <returns>Base64 representation of SpamSum $blocksize:$hash:$hash</returns>
public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash);
// Converts an ASCII null-terminated string to .NET string
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static string CToString(byte[] cString)
{
int count = 0;
// ReSharper disable once LoopCanBeConvertedToQuery
// LINQ is six times slower
foreach(byte c in cString)
{
if(c == 0)
break;
count++;
}
return Encoding.ASCII.GetString(cString, 0, count);
}
// Converts an ASCII null-terminated string to .NET string
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static byte[] CToArray(byte[] cString)
{
int count = 0;
// ReSharper disable once LoopCanBeConvertedToQuery
// LINQ is six times slower
foreach (byte c in cString)
{
if (c == 0)
break;
count++;
}
return new ReadOnlySpan<byte>(cString, 0, count).ToArray();
}
public void Dispose()
{
// TODO: No-op to dispose for now
}
struct RollState
{
public byte[] Window;
// ROLLING_WINDOW
public uint H1;
public uint H2;
public uint H3;
public uint N;
}
/* A blockhash contains a signature state for a specific (implicit) blocksize.
* The blocksize is given by SSDEEP_BS(index). The h and halfh members are the
* FNV hashes, where halfh stops to be reset after digest is SPAMSUM_LENGTH/2
* long. The halfh hash is needed be able to truncate digest for the second
* output hash to stay compatible with ssdeep output. */
struct BlockhashContext
{
public uint H;
public uint Halfh;
public byte[] Digest;
// SPAMSUM_LENGTH
public byte Halfdigest;
public uint Dlen;
}
struct FuzzyState
{
public uint Bhstart;
public uint Bhend;
public BlockhashContext[] Bh;
//NUM_BLOCKHASHES
public ulong TotalSize;
public RollState Roll;
}
}
}

View File

@@ -1,153 +0,0 @@
/*
Copyright (c) 2012-2015 Eugene Larchenko (spct@mail.ru)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
using System;
namespace SabreTools.Library.External
{
public class OptimizedCRC : IDisposable
{
private const uint kCrcPoly = 0xEDB88320;
private const uint kInitial = 0xFFFFFFFF;
private const int CRC_NUM_TABLES = 8;
private static readonly uint[] Table;
static OptimizedCRC()
{
unchecked
{
Table = new uint[256 * CRC_NUM_TABLES];
int i;
for (i = 0; i < 256; i++)
{
uint r = (uint)i;
for (int j = 0; j < 8; j++)
{
r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1));
}
Table[i] = r;
}
for (; i < 256 * CRC_NUM_TABLES; i++)
{
uint r = Table[i - 256];
Table[i] = Table[r & 0xFF] ^ (r >> 8);
}
}
}
public uint UnsignedValue;
public OptimizedCRC()
{
Init();
}
/// <summary>
/// Reset CRC
/// </summary>
public void Init()
{
UnsignedValue = kInitial;
}
public int Value
{
get { return (int)~UnsignedValue; }
}
public void Update(byte[] data, int offset, int count)
{
new ArraySegment<byte>(data, offset, count); // check arguments
if (count == 0)
{
return;
}
var table = OptimizedCRC.Table;
uint crc = UnsignedValue;
for (; (offset & 7) != 0 && count != 0; count--)
{
crc = (crc >> 8) ^ table[(byte)crc ^ data[offset++]];
}
if (count >= 8)
{
/*
* Idea from 7-zip project sources (http://7-zip.org/sdk.html)
*/
int end = (count - 8) & ~7;
count -= end;
end += offset;
while (offset != end)
{
crc ^= (uint)(data[offset] + (data[offset + 1] << 8) + (data[offset + 2] << 16) + (data[offset + 3] << 24));
uint high = (uint)(data[offset + 4] + (data[offset + 5] << 8) + (data[offset + 6] << 16) + (data[offset + 7] << 24));
offset += 8;
crc = table[(byte)crc + 0x700]
^ table[(byte)(crc >>= 8) + 0x600]
^ table[(byte)(crc >>= 8) + 0x500]
^ table[/*(byte)*/(crc >> 8) + 0x400]
^ table[(byte)(high) + 0x300]
^ table[(byte)(high >>= 8) + 0x200]
^ table[(byte)(high >>= 8) + 0x100]
^ table[/*(byte)*/(high >> 8) + 0x000];
}
}
while (count-- != 0)
{
crc = (crc >> 8) ^ table[(byte)crc ^ data[offset++]];
}
UnsignedValue = crc;
}
static public int Compute(byte[] data, int offset, int count)
{
var crc = new OptimizedCRC();
crc.Update(data, offset, count);
return crc.Value;
}
static public int Compute(byte[] data)
{
return Compute(data, 0, data.Length);
}
static public int Compute(ArraySegment<byte> block)
{
return Compute(block.Array, block.Offset, block.Count);
}
public void Dispose()
{
UnsignedValue = 0;
}
}
}

View File

@@ -2,6 +2,7 @@
using System.IO;
using System.Text;
using SabreTools.IO;
using SabreTools.Data;
using SabreTools.Library.FileTypes.Aaru;
using SabreTools.Library.IO;

View File

@@ -1,7 +1,13 @@
using System.IO;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Library.DatFiles;
using SabreTools.Data;
using SabreTools.IO;
using SabreTools.Library.IO;
using SabreTools.Logging;
using Compress.ThreadReaders;
namespace SabreTools.Library.FileTypes
{
@@ -135,7 +141,7 @@ namespace SabreTools.Library.FileTypes
if (getHashes)
{
BaseFile temp = stream.GetInfo(hashes: this.AvailableHashes);
BaseFile temp = GetInfo(stream, hashes: this.AvailableHashes);
if (temp != null)
{
this.Parent = temp.Parent;
@@ -156,5 +162,131 @@ namespace SabreTools.Library.FileTypes
}
#endregion
#region Static Methods
/// <summary>
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
{
// If we want to automatically set the size
if (size == -1)
size = input.Length;
try
{
// Get a list of hashers to run over the buffer
List<Hasher> hashers = new List<Hasher>();
if (hashes.HasFlag(Hash.CRC))
hashers.Add(new Hasher(Hash.CRC));
if (hashes.HasFlag(Hash.MD5))
hashers.Add(new Hasher(Hash.MD5));
#if NET_FRAMEWORK
if (hashes.HasFlag(Hash.RIPEMD160))
hashers.Add(new Hasher(Hash.RIPEMD160));
#endif
if (hashes.HasFlag(Hash.SHA1))
hashers.Add(new Hasher(Hash.SHA1));
if (hashes.HasFlag(Hash.SHA256))
hashers.Add(new Hasher(Hash.SHA256));
if (hashes.HasFlag(Hash.SHA384))
hashers.Add(new Hasher(Hash.SHA384));
if (hashes.HasFlag(Hash.SHA512))
hashers.Add(new Hasher(Hash.SHA512));
if (hashes.HasFlag(Hash.SpamSum))
hashers.Add(new Hasher(Hash.SpamSum));
// Initialize the hashing helpers
var loadBuffer = new ThreadLoadBuffer(input);
int buffersize = 3 * 1024 * 1024;
byte[] buffer0 = new byte[buffersize];
byte[] buffer1 = new byte[buffersize];
/*
Please note that some of the following code is adapted from
RomVault. This is a modified version of how RomVault does
threaded hashing. As such, some of the terminology and code
is the same, though variable names and comments may have
been tweaked to better fit this code base.
*/
// Pre load the first buffer
long refsize = size;
int next = refsize > buffersize ? buffersize : (int)refsize;
input.Read(buffer0, 0, next);
int current = next;
refsize -= next;
bool bufferSelect = true;
while (current > 0)
{
// Trigger the buffer load on the second buffer
next = refsize > buffersize ? buffersize : (int)refsize;
if (next > 0)
loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next);
byte[] buffer = bufferSelect ? buffer0 : buffer1;
// Run hashes in parallel
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current));
// Wait for the load buffer worker, if needed
if (next > 0)
loadBuffer.Wait();
// Setup for the next hashing step
current = next;
refsize -= next;
bufferSelect = !bufferSelect;
}
// Finalize all hashing helpers
loadBuffer.Finish();
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize());
// Get the results
BaseFile baseFile = new BaseFile()
{
Size = size,
CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null,
MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null,
#if NET_FRAMEWORK
RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null,
#endif
SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null,
SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null,
SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null,
SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null,
SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null,
};
// Dispose of the hashers
loadBuffer.Dispose();
hashers.ForEach(h => h.Dispose());
return baseFile;
}
catch (IOException ex)
{
LoggerImpl.Warning(ex, "An exception occurred during hashing.");
return new BaseFile();
}
finally
{
if (!keepReadOpen)
input.Dispose();
else
input.SeekIfPossible();
}
}
#endregion
}
}

View File

@@ -239,7 +239,7 @@ namespace SabreTools.Library.FileTypes
var gz = new gZip();
ZipReturn ret = gz.ZipFileOpen(this.Filename);
ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
gzipEntryRom = gzstream.GetInfo(hashes: this.AvailableHashes);
gzipEntryRom = GetInfo(gzstream, hashes: this.AvailableHashes);
gzipEntryRom.Filename = gz.Filename(0);
gzipEntryRom.Parent = gamename;
gzipEntryRom.Date = (gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);
@@ -461,7 +461,7 @@ namespace SabreTools.Library.FileTypes
outDir = Path.GetFullPath(outDir);
// Now get the Rom info for the file so we have hashes and size
rom = new Rom(inputStream.GetInfo(keepReadOpen: true));
rom = new Rom(GetInfo(inputStream, keepReadOpen: true));
// Get the output file name
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));

View File

@@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.DatFiles;
using SabreTools.Data;
using SabreTools.Library.DatItems;
using SabreTools.Library.IO;
using SharpCompress.Archives;
@@ -199,7 +199,7 @@ namespace SabreTools.Library.FileTypes
{
using (Stream entryStream = entry.OpenEntryStream())
{
rarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
rarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes);
}
}

View File

@@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.DatFiles;
using SabreTools.Data;
using SabreTools.Library.DatItems;
using SabreTools.Library.IO;
using SabreTools.Library.Tools;
@@ -307,7 +307,7 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
}
// Fill in comon details and add to the list

View File

@@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.DatFiles;
using SabreTools.Data;
using SabreTools.Library.DatItems;
using SabreTools.Library.IO;
using SabreTools.Library.Tools;
@@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes
{
using (Stream entryStream = entry.OpenEntryStream())
{
tarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
tarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes);
}
}

View File

@@ -229,7 +229,7 @@ namespace SabreTools.Library.FileTypes
else
{
var xzStream = new XZStream(File.OpenRead(this.Filename));
xzEntryRom = xzStream.GetInfo(hashes: this.AvailableHashes);
xzEntryRom = GetInfo(xzStream, hashes: this.AvailableHashes);
xzEntryRom.Filename = gamename;
xzStream.Dispose();
}
@@ -360,7 +360,7 @@ namespace SabreTools.Library.FileTypes
outDir = Path.GetFullPath(outDir);
// Now get the Rom info for the file so we have hashes and size
rom = new Rom(inputStream.GetInfo(keepReadOpen: true));
rom = new Rom(GetInfo(inputStream, keepReadOpen: true));
// Get the output file name
string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth));

View File

@@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.DatFiles;
using SabreTools.Data;
using SabreTools.Library.DatItems;
using SabreTools.Library.IO;
using SabreTools.Library.Tools;
@@ -308,7 +308,7 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
}
// Fill in comon details and add to the list

View File

@@ -391,7 +391,7 @@ namespace SabreTools.Library.IO
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
baseFile = CHDFile.Create(inputStream);
else
baseFile = inputStream.GetInfo(hashes: hashes, keepReadOpen: false);
baseFile = BaseFile.GetInfo(inputStream, hashes: hashes, keepReadOpen: false);
// Dispose of the input stream
inputStream?.Dispose();

View File

@@ -1,208 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Data;
using SabreTools.Logging;
using SabreTools.Library.DatFiles;
using SabreTools.Library.FileTypes;
using SabreTools.Library.Tools;
using Compress.ThreadReaders;
namespace SabreTools.Library.IO
{
/// <summary>
/// Extensions to Stream functionality
/// </summary>
public static class StreamExtensions
{
/// <summary>
/// Add an aribtrary number of bytes to the inputted stream
/// </summary>
/// <param name="input">Stream to be appended to</param>
/// <param name="output">Outputted stream</param>
/// <param name="bytesToAddToHead">Bytes to be added to head of stream</param>
/// <param name="bytesToAddToTail">Bytes to be added to tail of stream</param>
public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail)
{
// Write out prepended bytes
if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0)
output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length);
// Now copy the existing file over
input.CopyTo(output);
// Write out appended bytes
if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0)
output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length);
}
/// <summary>
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(this Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
{
return GetInfoAsync(input, size, hashes, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult();
}
/// <summary>
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
{
// If we want to automatically set the size
if (size == -1)
size = input.Length;
try
{
// Get a list of hashers to run over the buffer
List<Hasher> hashers = new List<Hasher>();
if (hashes.HasFlag(Hash.CRC))
hashers.Add(new Hasher(Hash.CRC));
if (hashes.HasFlag(Hash.MD5))
hashers.Add(new Hasher(Hash.MD5));
#if NET_FRAMEWORK
if (hashes.HasFlag(Hash.RIPEMD160))
hashers.Add(new Hasher(Hash.RIPEMD160));
#endif
if (hashes.HasFlag(Hash.SHA1))
hashers.Add(new Hasher(Hash.SHA1));
if (hashes.HasFlag(Hash.SHA256))
hashers.Add(new Hasher(Hash.SHA256));
if (hashes.HasFlag(Hash.SHA384))
hashers.Add(new Hasher(Hash.SHA384));
if (hashes.HasFlag(Hash.SHA512))
hashers.Add(new Hasher(Hash.SHA512));
if (hashes.HasFlag(Hash.SpamSum))
hashers.Add(new Hasher(Hash.SpamSum));
// Initialize the hashing helpers
var loadBuffer = new ThreadLoadBuffer(input);
int buffersize = 3 * 1024 * 1024;
byte[] buffer0 = new byte[buffersize];
byte[] buffer1 = new byte[buffersize];
/*
Please note that some of the following code is adapted from
RomVault. This is a modified version of how RomVault does
threaded hashing. As such, some of the terminology and code
is the same, though variable names and comments may have
been tweaked to better fit this code base.
*/
// Pre load the first buffer
long refsize = size;
int next = refsize > buffersize ? buffersize : (int)refsize;
input.Read(buffer0, 0, next);
int current = next;
refsize -= next;
bool bufferSelect = true;
while (current > 0)
{
// Trigger the buffer load on the second buffer
next = refsize > buffersize ? buffersize : (int)refsize;
if (next > 0)
loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next);
byte[] buffer = bufferSelect ? buffer0 : buffer1;
// Run hashes in parallel
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current));
// Wait for the load buffer worker, if needed
if (next > 0)
loadBuffer.Wait();
// Setup for the next hashing step
current = next;
refsize -= next;
bufferSelect = !bufferSelect;
}
// Finalize all hashing helpers
loadBuffer.Finish();
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize());
// Get the results
BaseFile baseFile = new BaseFile()
{
Size = size,
CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null,
MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null,
#if NET_FRAMEWORK
RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null,
#endif
SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null,
SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null,
SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null,
SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null,
SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null,
};
// Dispose of the hashers
loadBuffer.Dispose();
hashers.ForEach(h => h.Dispose());
return baseFile;
}
catch (IOException ex)
{
LoggerImpl.Warning(ex, "An exception occurred during hashing.");
return new BaseFile();
}
finally
{
if (!keepReadOpen)
input.Dispose();
else
input.SeekIfPossible();
}
}
/// <summary>
/// Seek to a specific point in the stream, if possible
/// </summary>
/// <param name="input">Input stream to try seeking on</param>
/// <param name="offset">Optional offset to seek to</param>
public static long SeekIfPossible(this Stream input, long offset = 0)
{
try
{
if (input.CanSeek)
{
if (offset < 0)
return input.Seek(offset, SeekOrigin.End);
else if (offset >= 0)
return input.Seek(offset, SeekOrigin.Begin);
}
return input.Position;
}
catch (NotSupportedException ex)
{
LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed");
}
catch (NotImplementedException ex)
{
LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed");
}
return -1;
}
}
}

View File

@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using SabreTools.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.FileTypes;

View File

@@ -1,157 +0,0 @@
using System;
using System.Linq;
using System.Security.Cryptography;
using Aaru.Checksums;
using SabreTools.Library.DatFiles;
using SabreTools.Library.External;
namespace SabreTools.Library.Tools
{
/// <summary>
/// Async hashing class wraper
/// </summary>
public class Hasher
{
public Hash HashType { get; private set; }
private IDisposable _hasher;
public Hasher(Hash hashType)
{
this.HashType = hashType;
GetHasher();
}
/// <summary>
/// Generate the correct hashing class based on the hash type
/// </summary>
private void GetHasher()
{
switch (HashType)
{
case Hash.CRC:
_hasher = new OptimizedCRC();
break;
case Hash.MD5:
_hasher = MD5.Create();
break;
#if NET_FRAMEWORK
case Hash.RIPEMD160:
_hasher = RIPEMD160.Create();
break;
#endif
case Hash.SHA1:
_hasher = SHA1.Create();
break;
case Hash.SHA256:
_hasher = SHA256.Create();
break;
case Hash.SHA384:
_hasher = SHA384.Create();
break;
case Hash.SHA512:
_hasher = SHA512.Create();
break;
case Hash.SpamSum:
_hasher = new SpamSumContext();
break;
}
}
public void Dispose()
{
_hasher.Dispose();
}
/// <summary>
/// Process a buffer of some length with the internal hash algorithm
/// </summary>
public void Process(byte[] buffer, int size)
{
switch (HashType)
{
case Hash.CRC:
(_hasher as OptimizedCRC).Update(buffer, 0, size);
break;
case Hash.MD5:
#if NET_FRAMEWORK
case Hash.RIPEMD160:
#endif
case Hash.SHA1:
case Hash.SHA256:
case Hash.SHA384:
case Hash.SHA512:
(_hasher as HashAlgorithm).TransformBlock(buffer, 0, size, null, 0);
break;
case Hash.SpamSum:
(_hasher as SpamSumContext).Update(buffer);
break;
}
}
/// <summary>
/// Finalize the internal hash algorigthm
/// </summary>
public void Finalize()
{
byte[] emptyBuffer = new byte[0];
switch (HashType)
{
case Hash.CRC:
(_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0);
break;
case Hash.MD5:
#if NET_FRAMEWORK
case Hash.RIPEMD160:
#endif
case Hash.SHA1:
case Hash.SHA256:
case Hash.SHA384:
case Hash.SHA512:
(_hasher as HashAlgorithm).TransformFinalBlock(emptyBuffer, 0, 0);
break;
case Hash.SpamSum:
// No finalization step needed
break;
}
}
/// <summary>
/// Get internal hash as a byte array
/// </summary>
public byte[] GetHash()
{
switch (HashType)
{
case Hash.CRC:
return BitConverter.GetBytes((_hasher as OptimizedCRC).Value).Reverse().ToArray();
case Hash.MD5:
#if NET_FRAMEWORK
case Hash.RIPEMD160:
#endif
case Hash.SHA1:
case Hash.SHA256:
case Hash.SHA384:
case Hash.SHA512:
return (_hasher as HashAlgorithm).Hash;
case Hash.SpamSum:
return (_hasher as SpamSumContext).Final();
}
return null;
}
}
}