From 3fa04bba65695252f2a028ef5808f2558be7e222 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 7 Nov 2024 21:53:07 -0500 Subject: [PATCH] Add most XXH64 code --- SabreTools.Hashing/XxHash/Utility.cs | 83 ++++++++- SabreTools.Hashing/XxHash/XXH32State.cs | 2 - SabreTools.Hashing/XxHash/XXH64State.cs | 231 +++++++++++++++++++++--- 3 files changed, 286 insertions(+), 30 deletions(-) diff --git a/SabreTools.Hashing/XxHash/Utility.cs b/SabreTools.Hashing/XxHash/Utility.cs index 5fb19e1..ea93300 100644 --- a/SabreTools.Hashing/XxHash/Utility.cs +++ b/SabreTools.Hashing/XxHash/Utility.cs @@ -20,8 +20,8 @@ namespace SabreTools.Hashing.XxHash public static uint Swap32(uint x) { return ((x << 24) & 0xff000000) - | ((x << 8 ) & 0x00ff0000) - | ((x >> 8 ) & 0x0000ff00) + | ((x << 8) & 0x00ff0000) + | ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); } @@ -58,8 +58,85 @@ namespace SabreTools.Hashing.XxHash return value; } - + public static uint XXH_get32bits(byte[] data, int offset, Alignment align) => ReadLE32Align(data, offset, align); + + public static ulong Read64(byte[] data, int offset) + { + return (ulong)data[offset + 7] + | data[offset + 6] + | data[offset + 5] + | data[offset + 4] + | data[offset + 3] + | data[offset + 2] + | data[offset + 1] + | data[offset + 0]; + } + + public static ulong Swap64(ulong x) + { + return ((x << 56) & 0xff00000000000000) + | ((x << 40) & 0x00ff000000000000) + | ((x << 24) & 0x0000ff0000000000) + | ((x << 8) & 0x000000ff00000000) + | ((x >> 8) & 0x00000000ff000000) + | ((x >> 24) & 0x0000000000ff0000) + | ((x >> 40) & 0x000000000000ff00) + | ((x >> 56) & 0x00000000000000ff); + } + + public static ulong ReadLE64(byte[] data, int offset) + { + return (ulong)data[offset + 0] + | data[offset + 1] + | data[offset + 2] + | data[offset + 3] + | data[offset + 4] + | data[offset + 5] + | data[offset + 6] + | data[offset + 7]; + } + + public static ulong ReadBE64(byte[] data, int offset) + { + return (ulong)data[offset + 7] + | data[offset + 6] + | data[offset + 5] + | data[offset + 4] + | data[offset + 3] + | data[offset + 2] + | data[offset + 1] + | data[offset + 0]; + } + + public static ulong ReadLE64Align(byte[] data, int offset, Alignment align) + { + if (align == Alignment.XXH_aligned) + return ReadLE64(data, offset); + + ulong value = 0; + if (offset + 0 < data.Length) + value |= data[offset + 0]; + if (offset + 1 < data.Length) + value |= data[offset + 1]; + if (offset + 2 < data.Length) + value |= data[offset + 2]; + if (offset + 3 < data.Length) + value |= data[offset + 3]; + if (offset + 4 < data.Length) + value |= data[offset + 4]; + if (offset + 5 < data.Length) + value |= data[offset + 5]; + if (offset + 6 < data.Length) + value |= data[offset + 6]; + if (offset + 7 < data.Length) + value |= data[offset + 7]; + + return value; + } + + public static ulong XXH_get64bits(byte[] data, int offset, Alignment align) + => ReadLE64Align(data, offset, align); } } \ No newline at end of file diff --git a/SabreTools.Hashing/XxHash/XXH32State.cs b/SabreTools.Hashing/XxHash/XXH32State.cs index cb98160..d2ed1e9 100644 --- a/SabreTools.Hashing/XxHash/XXH32State.cs +++ b/SabreTools.Hashing/XxHash/XXH32State.cs @@ -167,8 +167,6 @@ namespace SabreTools.Hashing.XxHash /// The final mix ensures that all input bits have a chance to impact any bit in /// the output digest, resulting in an unbiased distribution. /// - /// - /// private static uint Avalanche(uint hash) { hash ^= hash >> 15; diff --git a/SabreTools.Hashing/XxHash/XXH64State.cs b/SabreTools.Hashing/XxHash/XXH64State.cs index 949964a..3c280e5 100644 --- a/SabreTools.Hashing/XxHash/XXH64State.cs +++ b/SabreTools.Hashing/XxHash/XXH64State.cs @@ -1,3 +1,7 @@ +using System; +using static SabreTools.Hashing.XxHash.Constants; +using static SabreTools.Hashing.XxHash.Utility; + namespace SabreTools.Hashing.XxHash { /// @@ -14,17 +18,17 @@ namespace SabreTools.Hashing.XxHash /// /// Accumulator lanes /// - public ulong[] AccumulatorLanes { get; } = new ulong[4]; + public ulong[] V { get; } = new ulong[4]; /// /// Internal buffer for partial reads. Treated as unsigned char[16]. /// - public ulong[] PartialReadBuffer { get; } = new ulong[4]; + public byte[] Memory { get; } = new byte[16]; /// - /// Amount of data in + /// Amount of data in /// - public uint Memsize { get; set; } + public int Memsize { get; set; } /// /// Reserved field, needed for padding anyways @@ -36,31 +40,29 @@ namespace SabreTools.Hashing.XxHash /// public ulong Reserved64 { get; set; } - /// - /// The 64-bit seed to alter the hash's output predictably. - /// - private ulong _seed; - - public XXH64State() - { - _seed = 0; - } - - /// The 64-bit seed to alter the hash result predictably. - public XXH64State(ulong seed) - { - _seed = seed; - } - /// /// Resets to begin a new hash /// /// The 64-bit seed to alter the hash result predictably. public void Reset(ulong seed) { - // TODO: XXH64_reset function + TotalLength = default; + + V[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + V[1] = seed + XXH_PRIME64_2; + V[2] = seed + 0; + V[3] = seed - XXH_PRIME64_1; + + for (int i = 0; i < Memory.Length; i++) + { + Memory[i] = default; + } + + Memsize = default; + Reserved32 = default; + Reserved64 = default; } - + /// /// Hash a block of data and append it to the existing hash /// @@ -69,7 +71,48 @@ namespace SabreTools.Hashing.XxHash /// Length of the data to hash public void TransformBlock(byte[] data, int offset, int length) { - // TODO: XXH64_update function + int bEnd = offset + length; + + TotalLength += (ulong)length; + + if (Memsize + length < 32) + { + // Fill in tmp buffer + Array.Copy(data, offset, Memory, Memsize, length); + Memsize += length; + return; + } + + if (Memsize > 0) + { + // tmp buffer is full + Array.Copy(data, offset, Memory, Memsize, 32 - Memsize); + V[0] = Round(V[0], ReadLE64(Memory, 0)); + V[1] = Round(V[1], ReadLE64(Memory, 1)); + V[2] = Round(V[2], ReadLE64(Memory, 2)); + V[3] = Round(V[3], ReadLE64(Memory, 3)); + offset += 32 - Memsize; + Memsize = 0; + } + + if (offset + 32 <= bEnd) + { + int limit = bEnd - 32; + + do + { + V[0] = Round(V[0], ReadLE64(data, offset)); offset += 8; + V[1] = Round(V[1], ReadLE64(data, offset)); offset += 8; + V[2] = Round(V[2], ReadLE64(data, offset)); offset += 8; + V[3] = Round(V[3], ReadLE64(data, offset)); offset += 8; + } while (offset <= limit); + } + + if (offset < bEnd) + { + Array.Copy(data, offset, Memory, 0, bEnd - offset); + Memsize = bEnd - offset; + } } /// @@ -78,8 +121,146 @@ namespace SabreTools.Hashing.XxHash /// The calculated 64-bit xxHash64 value from that state. public ulong Digest() { - // TODO: XXH64_update - return ulong.MaxValue; + ulong h64; + + if (TotalLength >= 32) + { + h64 = XXH_rotl64(V[0], 1) + XXH_rotl64(V[1], 7) + XXH_rotl64(V[2], 12) + XXH_rotl64(V[3], 18); + h64 = MergeRound(h64, V[0]); + h64 = MergeRound(h64, V[1]); + h64 = MergeRound(h64, V[2]); + h64 = MergeRound(h64, V[3]); + } + else + { + h64 = V[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += TotalLength; + + return Finalize(h64, Memory, 0, (int)TotalLength, Alignment.XXH_aligned); + } + + /// + /// Normal stripe processing routine. + /// + /// This shuffles the bits so that any bit from @p input impacts + /// several bits in @p acc. + /// + /// The accumulator lane. + /// The stripe of input to mix. + /// The mixed accumulator lane. + private static ulong Round(ulong acc, ulong input) + { + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; + } + + private static ulong MergeRound(ulong acc, ulong val) + { + val = Round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; + } + + /// + /// Mixes all bits to finalize the hash. + /// + /// The final mix ensures that all input bits have a chance to impact any bit in + /// the output digest, resulting in an unbiased distribution. + /// + private static ulong Avalanche(ulong hash) + { + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; + } + + /// + /// Processes the last 0-31 bytes of @p ptr. + /// + /// There may be up to 31 bytes remaining to consume from the input. + /// This final stage will digest them to ensure that all input bytes are present + /// in the final mix. + /// + /// The hash to finalize. + /// The pointer to the remaining input. + /// The pointer to the remaining input. + /// The remaining length, modulo 32. + /// Whether @p ptr is aligned. + /// The finalized hash + private static ulong Finalize(ulong hash, byte[] data, int offset, int length, Alignment align) + { + length &= 31; + while (length >= 8) + { + ulong k1 = Round(0, XXH_get64bits(data, offset, align)); + offset += 8; + hash ^= k1; + hash = XXH_rotl64(hash, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; + length -= 8; + } + if (length >= 4) + { + hash ^= (ulong)(XXH_get32bits(data, offset, align)) * XXH_PRIME64_1; + offset += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + length -= 4; + } + while (length > 0) + { + hash ^= data[offset++] * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --length; + } + return Avalanche(hash); + } + + /// + /// The implementation for XXH64 + /// + /// The calculated hash. + private static ulong EndianAlign(byte[] data, int offset, int length, ulong seed, Alignment align) + { + ulong h64; + if (length >= 32) + { + int bEnd = offset + length; + int limit = bEnd - 31; + ulong v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + ulong v2 = seed + XXH_PRIME64_2; + ulong v3 = seed + 0; + ulong v4 = seed - XXH_PRIME64_1; + + do + { + v1 = Round(v1, XXH_get64bits(data, offset, align)); offset += 8; + v2 = Round(v2, XXH_get64bits(data, offset, align)); offset += 8; + v3 = Round(v3, XXH_get64bits(data, offset, align)); offset += 8; + v4 = Round(v4, XXH_get64bits(data, offset, align)); offset += 8; + } while (offset < limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = MergeRound(h64, v1); + h64 = MergeRound(h64, v2); + h64 = MergeRound(h64, v3); + h64 = MergeRound(h64, v4); + + } + else + { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (ulong)length; + + return Finalize(h64, data, offset, length, align); } } } \ No newline at end of file