claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support ancient .NET in Core

Browse Source
This commit is contained in:
Matt Nadareski
2024-02-28 19:49:09 -05:00
parent 823a9ca7b7
commit 778399cad7
11 changed files with 269 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
SabreTools.Core/Tools/Aaru/SpamSumContext.cs
View File

@@ -49,12 +49,12 @@ namespace Aaru.Checksums
/// <summary>Implements the SpamSum fuzzy hashing algorithm.</summary>
public sealed class SpamSumContext : IChecksum, IDisposable
{
const uint ROLLING_WINDOW = 7;
const uint MIN_BLOCKSIZE = 3;
const uint HASH_PRIME = 0x01000193;
const uint HASH_INIT = 0x28021967;
const uint NUM_BLOCKHASHES = 31;
const uint SPAMSUM_LENGTH = 64;
const uint ROLLING_WINDOW = 7;
const uint MIN_BLOCKSIZE = 3;
const uint HASH_PRIME = 0x01000193;
const uint HASH_INIT = 0x28021967;
const uint NUM_BLOCKHASHES = 31;
const uint SPAMSUM_LENGTH = 64;
const uint FUZZY_MAX_RESULT = (2 * SPAMSUM_LENGTH) + 20;
//"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
@@ -76,17 +76,17 @@ namespace Aaru.Checksums
Bh = new BlockhashContext[NUM_BLOCKHASHES]
};
for(int i = 0; i < NUM_BLOCKHASHES; i++)
for (int i = 0; i < NUM_BLOCKHASHES; i++)
_self.Bh[i].Digest = new byte[SPAMSUM_LENGTH];
_self.Bhstart = 0;
_self.Bhend = 1;
_self.Bh[0].H = HASH_INIT;
_self.Bh[0].Halfh = HASH_INIT;
_self.Bh[0].Digest[0] = 0;
_self.Bhstart = 0;
_self.Bhend = 1;
_self.Bh[0].H = HASH_INIT;
_self.Bh[0].Halfh = HASH_INIT;
_self.Bh[0].Digest[0] = 0;
_self.Bh[0].Halfdigest = 0;
_self.Bh[0].Dlen = 0;
_self.TotalSize = 0;
_self.Bh[0].Dlen = 0;
_self.TotalSize = 0;
roll_init();
}
@@ -98,7 +98,7 @@ namespace Aaru.Checksums
{
_self.TotalSize += len;
for(int i = 0; i < len; i++)
for (int i = 0; i < len; i++)
fuzzy_engine_step(data[i]);
}
@@ -125,7 +125,9 @@ namespace Aaru.Checksums
return CToString(result);
}
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void roll_init() => _self.Roll = new RollState
{
Window = new byte[ROLLING_WINDOW]
@@ -141,7 +143,9 @@ namespace Aaru.Checksums
* h3 is a shift/xor based rolling hash, and is mostly needed to ensure that
* we can cope with large blocksize values
*/
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void roll_hash(byte c)
{
_self.Roll.H2 -= _self.Roll.H1;
@@ -157,54 +161,64 @@ namespace Aaru.Checksums
* in theory should have no effect. This AND has been removed
* for performance (jk) */
_self.Roll.H3 <<= 5;
_self.Roll.H3 ^= c;
_self.Roll.H3 ^= c;
}
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
uint roll_sum() => _self.Roll.H1 + _self.Roll.H2 + _self.Roll.H3;
/* A simple non-rolling hash, based on the FNV hash. */
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
static uint sum_hash(byte c, uint h) => (h * HASH_PRIME) ^ c;
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
static uint SSDEEP_BS(uint index) => MIN_BLOCKSIZE << (int)index;
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void fuzzy_try_fork_blockhash()
{
if(_self.Bhend >= NUM_BLOCKHASHES)
if (_self.Bhend >= NUM_BLOCKHASHES)
return;
if(_self.Bhend == 0) // assert
if (_self.Bhend == 0) // assert
throw new Exception("Assertion failed");
uint obh = _self.Bhend - 1;
uint nbh = _self.Bhend;
_self.Bh[nbh].H = _self.Bh[obh].H;
_self.Bh[nbh].Halfh = _self.Bh[obh].Halfh;
_self.Bh[nbh].Digest[0] = 0;
_self.Bh[nbh].H = _self.Bh[obh].H;
_self.Bh[nbh].Halfh = _self.Bh[obh].Halfh;
_self.Bh[nbh].Digest[0] = 0;
_self.Bh[nbh].Halfdigest = 0;
_self.Bh[nbh].Dlen = 0;
_self.Bh[nbh].Dlen = 0;
++_self.Bhend;
}
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void fuzzy_try_reduce_blockhash()
{
if(_self.Bhstart >= _self.Bhend)
if (_self.Bhstart >= _self.Bhend)
throw new Exception("Assertion failed");
if(_self.Bhend - _self.Bhstart < 2)
if (_self.Bhend - _self.Bhstart < 2)
/* Need at least two working hashes. */
return;
if((ulong)SSDEEP_BS(_self.Bhstart) * SPAMSUM_LENGTH >= _self.TotalSize)
if ((ulong)SSDEEP_BS(_self.Bhstart) * SPAMSUM_LENGTH >= _self.TotalSize)
/* Initial blocksize estimate would select this or a smaller
* blocksize. */
return;
if(_self.Bh[_self.Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2)
if (_self.Bh[_self.Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2)
/* Estimate adjustment would select this blocksize. */
return;
@@ -213,7 +227,9 @@ namespace Aaru.Checksums
++_self.Bhstart;
}
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void fuzzy_engine_step(byte c)
{
uint i;
@@ -223,16 +239,16 @@ namespace Aaru.Checksums
roll_hash(c);
ulong h = roll_sum();
for(i = _self.Bhstart; i < _self.Bhend; ++i)
for (i = _self.Bhstart; i < _self.Bhend; ++i)
{
_self.Bh[i].H = sum_hash(c, _self.Bh[i].H);
_self.Bh[i].H = sum_hash(c, _self.Bh[i].H);
_self.Bh[i].Halfh = sum_hash(c, _self.Bh[i].Halfh);
}
for(i = _self.Bhstart; i < _self.Bhend; ++i)
for (i = _self.Bhstart; i < _self.Bhend; ++i)
{
/* With growing blocksize almost no runs fail the next test. */
if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
if (h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
/* Once this condition is false for one bs, it is
* automatically false for all further bs. I.e. if
* h === -1 (mod 2*bs) then h === -1 (mod bs). */
@@ -241,13 +257,13 @@ namespace Aaru.Checksums
/* We have hit a reset point. We now emit hashes which are
* based on all characters in the piece of the message between
* the last reset point and this one */
if(0 == _self.Bh[i].Dlen)
if (0 == _self.Bh[i].Dlen)
fuzzy_try_fork_blockhash();
_self.Bh[i].Digest[_self.Bh[i].Dlen] = _b64[_self.Bh[i].H % 64];
_self.Bh[i].Halfdigest = _b64[_self.Bh[i].Halfh % 64];
_self.Bh[i].Digest[_self.Bh[i].Dlen] = _b64[_self.Bh[i].H % 64];
_self.Bh[i].Halfdigest = _b64[_self.Bh[i].Halfh % 64];
if(_self.Bh[i].Dlen < SPAMSUM_LENGTH - 1)
if (_self.Bh[i].Dlen < SPAMSUM_LENGTH - 1)
{
/* We can have a problem with the tail overflowing. The
* easiest way to cope with this is to only reset the
@@ -256,12 +272,12 @@ namespace Aaru.Checksums
* last few pieces of the message into a single piece
* */
_self.Bh[i].Digest[++_self.Bh[i].Dlen] = 0;
_self.Bh[i].H = HASH_INIT;
_self.Bh[i].H = HASH_INIT;
if(_self.Bh[i].Dlen >= SPAMSUM_LENGTH / 2)
if (_self.Bh[i].Dlen >= SPAMSUM_LENGTH / 2)
continue;
_self.Bh[i].Halfh = HASH_INIT;
_self.Bh[i].Halfh = HASH_INIT;
_self.Bh[i].Halfdigest = 0;
}
else
@@ -270,50 +286,52 @@ namespace Aaru.Checksums
}
// CLAUNIA: Flags seems to never be used in ssdeep, so I just removed it for code simplicity
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
void FuzzyDigest(out byte[] result)
{
var sb = new StringBuilder();
uint bi = _self.Bhstart;
uint h = roll_sum();
int remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */
var sb = new StringBuilder();
uint bi = _self.Bhstart;
uint h = roll_sum();
int remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */
result = new byte[FUZZY_MAX_RESULT];
/* Verify that our elimination was not overeager. */
if(!(bi == 0 || ((ulong)SSDEEP_BS(bi) / 2) * SPAMSUM_LENGTH < _self.TotalSize))
if (!(bi == 0 || ((ulong)SSDEEP_BS(bi) / 2) * SPAMSUM_LENGTH < _self.TotalSize))
throw new Exception("Assertion failed");
int resultOff = 0;
/* Initial blocksize guess. */
while((ulong)SSDEEP_BS(bi) * SPAMSUM_LENGTH < _self.TotalSize)
while ((ulong)SSDEEP_BS(bi) * SPAMSUM_LENGTH < _self.TotalSize)
{
++bi;
if(bi >= NUM_BLOCKHASHES)
if (bi >= NUM_BLOCKHASHES)
throw new OverflowException("The input exceeds data types.");
}
/* Adapt blocksize guess to actual digest length. */
while(bi >= _self.Bhend)
while (bi >= _self.Bhend)
--bi;
while(bi > _self.Bhstart &&
while (bi > _self.Bhstart &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
--bi;
if(bi > 0 &&
if (bi > 0 &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
throw new Exception("Assertion failed");
sb.AppendFormat("{0}:", SSDEEP_BS(bi));
int i = Encoding.ASCII.GetBytes(sb.ToString()).Length;
if(i <= 0)
if (i <= 0)
/* Maybe snprintf has set errno here? */
throw new OverflowException("The input exceeds data types.");
if(i >= remain)
if (i >= remain)
throw new Exception("Assertion failed");
remain -= i;
@@ -324,21 +342,21 @@ namespace Aaru.Checksums
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
if (i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
remain -= i;
if(h != 0)
if (h != 0)
{
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _b64[_self.Bh[bi].H % 64];
if(i < 3 ||
if (i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
@@ -347,14 +365,14 @@ namespace Aaru.Checksums
--remain;
}
}
else if(_self.Bh[bi].Digest[i] != 0)
else if (_self.Bh[bi].Digest[i] != 0)
{
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _self.Bh[bi].Digest[i];
if(i < 3 ||
if (i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
@@ -364,33 +382,33 @@ namespace Aaru.Checksums
}
}
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = 0x3A; // ':'
--remain;
if(bi < _self.Bhend - 1)
if (bi < _self.Bhend - 1)
{
++bi;
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
if (i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
remain -= i;
if(h != 0)
if (h != 0)
{
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
h = _self.Bh[bi].Halfh;
h = _self.Bh[bi].Halfh;
result[resultOff] = _b64[h % 64];
if(i < 3 ||
if (i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
@@ -403,14 +421,14 @@ namespace Aaru.Checksums
{
i = _self.Bh[bi].Halfdigest;
if(i != 0)
if (i != 0)
{
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = (byte)i;
if(i < 3 ||
if (i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
@@ -421,12 +439,12 @@ namespace Aaru.Checksums
}
}
}
else if(h != 0)
else if (h != 0)
{
if(_self.Bh[bi].Dlen != 0)
if (_self.Bh[bi].Dlen != 0)
throw new Exception("Assertion failed");
if(remain <= 0)
if (remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = _b64[_self.Bh[bi].H % 64];
@@ -472,16 +490,18 @@ namespace Aaru.Checksums
public static string Data(byte[] data, out byte[]? hash) => Data(data, (uint)data.Length, out hash);
// Converts an ASCII null-terminated string to .NET string
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
static string CToString(byte[] cString)
{
int count = 0;
// ReSharper disable once LoopCanBeConvertedToQuery
// LINQ is six times slower
foreach(byte c in cString)
foreach (byte c in cString)
{
if(c == 0)
if (c == 0)
break;
count++;
@@ -491,7 +511,9 @@ namespace Aaru.Checksums
}
// Converts an ASCII null-terminated string to .NET string
#if NET452_OR_GREATER || NETCOREAPP
[MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
static byte[] CToArray(byte[] cString)
{
int count = 0;
@@ -506,7 +528,13 @@ namespace Aaru.Checksums
count++;
}
#if NETFRAMEWORK
byte[] temp = new byte[count];
Array.Copy(cString, temp, count);
return temp;
#else
return new ReadOnlySpan<byte>(cString, 0, count).ToArray();
#endif
}
public void Dispose()
@@ -532,8 +560,8 @@ namespace Aaru.Checksums
* output hash to stay compatible with ssdeep output. */
struct BlockhashContext
{
public uint H;
public uint Halfh;
public uint H;
public uint Halfh;
public byte[] Digest;
// SPAMSUM_LENGTH
@@ -543,12 +571,12 @@ namespace Aaru.Checksums
struct FuzzyState
{
public uint Bhstart;
public uint Bhend;
public uint Bhstart;
public uint Bhend;
public BlockhashContext[] Bh;
//NUM_BLOCKHASHES
public ulong TotalSize;
public ulong TotalSize;
public RollState Roll;
}
}