From 477a30cf5bf2ed5c4e8687269e69a5c31f78489f Mon Sep 17 00:00:00 2001 From: Jason Nelson Date: Wed, 18 Nov 2020 11:21:29 -0800 Subject: [PATCH] Use hardware accelerated Alder32 impl --- src/SharpCompress/Algorithms/Alder32.cs | 285 ++++++++++++++++++ .../Compressors/Deflate/DeflateManager.cs | 24 +- .../Compressors/Deflate/Inflate.cs | 12 +- src/SharpCompress/Compressors/Deflate/Zlib.cs | 84 ------ .../Compressors/Deflate/ZlibCodec.cs | 6 +- 5 files changed, 308 insertions(+), 103 deletions(-) create mode 100644 src/SharpCompress/Algorithms/Alder32.cs diff --git a/src/SharpCompress/Algorithms/Alder32.cs b/src/SharpCompress/Algorithms/Alder32.cs new file mode 100644 index 00000000..d0b3279f --- /dev/null +++ b/src/SharpCompress/Algorithms/Alder32.cs @@ -0,0 +1,285 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the GNU Affero General Public License, Version 3. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +#if NETCOREAPP3_1 +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +#endif + +namespace SharpCompress.Algorithms +{ + /// + /// Calculates the 32 bit Adler checksum of a given buffer according to + /// RFC 1950. ZLIB Compressed Data Format Specification version 3.3) + /// + internal static class Adler32 + { + /// + /// The default initial seed value of a Adler32 checksum calculation. + /// + public const uint SeedValue = 1U; + +#if NETCOREAPP3_1 + private const int MinBufferSize = 64; +#endif + + // Largest prime smaller than 65536 + private const uint BASE = 65521; + + // NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 + private const uint NMAX = 5552; + + /// + /// Calculates the Adler32 checksum with the bytes taken from the span. + /// + /// The readonly span of bytes. + /// The . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint Calculate(ReadOnlySpan buffer) + { + return Calculate(SeedValue, buffer); + } + + /// + /// Calculates the Adler32 checksum with the bytes taken from the span and seed. + /// + /// The input Adler32 value. + /// The readonly span of bytes. + /// The . + public static uint Calculate(uint adler, ReadOnlySpan buffer) + { + if (buffer.IsEmpty) + { + return SeedValue; + } + +#if NETCOREAPP3_1 + if (Sse3.IsSupported && buffer.Length >= MinBufferSize) + { + return CalculateSse(adler, buffer); + } + + return CalculateScalar(adler, buffer); +#else + return CalculateScalar(adler, buffer); +#endif + } + + // Based on https://github.com/chromium/chromium/blob/master/third_party/zlib/adler32_simd.c +#if NETCOREAPP3_1 + private static unsafe uint CalculateSse(uint adler, ReadOnlySpan buffer) + { + uint s1 = adler & 0xFFFF; + uint s2 = (adler >> 16) & 0xFFFF; + + // Process the data in blocks. + const int BLOCK_SIZE = 1 << 5; + + uint length = (uint)buffer.Length; + uint blocks = length / BLOCK_SIZE; + length -= blocks * BLOCK_SIZE; + + int index = 0; + fixed (byte* bufferPtr = &buffer[0]) + { + index += (int)blocks * BLOCK_SIZE; + var localBufferPtr = bufferPtr; + + // _mm_setr_epi8 on x86 + var tap1 = Vector128.Create(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17); + var tap2 = Vector128.Create(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + Vector128 zero = Vector128.Zero; + var ones = Vector128.Create((short)1); + + while (blocks > 0) + { + uint n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + { + n = blocks; + } + + blocks -= n; + + // Process n blocks of data. At most NMAX data bytes can be + // processed before s2 must be reduced modulo BASE. + Vector128 v_ps = Vector128.CreateScalar(s1 * n).AsInt32(); + Vector128 v_s2 = Vector128.CreateScalar(s2).AsInt32(); + Vector128 v_s1 = Vector128.Zero; + + do + { + // Load 32 input bytes. + Vector128 bytes1 = Sse3.LoadDquVector128(localBufferPtr); + Vector128 bytes2 = Sse3.LoadDquVector128(localBufferPtr + 16); + + // Add previous block byte sum to v_ps. + v_ps = Sse2.Add(v_ps, v_s1); + + // Horizontally add the bytes for s1, multiply-adds the + // bytes by [ 32, 31, 30, ... ] for s2. + v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes1, zero).AsInt32()); + Vector128 mad1 = Ssse3.MultiplyAddAdjacent(bytes1, tap1); + v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad1, ones)); + + v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes2, zero).AsInt32()); + Vector128 mad2 = Ssse3.MultiplyAddAdjacent(bytes2, tap2); + v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad2, ones)); + + localBufferPtr += BLOCK_SIZE; + } + while (--n > 0); + + v_s2 = Sse2.Add(v_s2, Sse2.ShiftLeftLogical(v_ps, 5)); + + // Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + const byte S2301 = 0b1011_0001; // A B C D -> B A D C + const byte S1032 = 0b0100_1110; // A B C D -> C D A B + + v_s1 = Sse2.Add(v_s1, Sse2.Shuffle(v_s1, S2301)); + v_s1 = Sse2.Add(v_s1, Sse2.Shuffle(v_s1, S1032)); + + s1 += (uint)v_s1.ToScalar(); + + v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S2301)); + v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S1032)); + + s2 = (uint)v_s2.ToScalar(); + + // Reduce. + s1 %= BASE; + s2 %= BASE; + } + } + + ref byte bufferRef = ref MemoryMarshal.GetReference(buffer); + + if (length > 0) + { + if (length >= 16) + { + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + length -= 16; + } + + while (length-- > 0) + { + s2 += s1 += Unsafe.Add(ref bufferRef, index++); + } + + if (s1 >= BASE) + { + s1 -= BASE; + } + + s2 %= BASE; + } + + return s1 | (s2 << 16); + } +#endif + + private static uint CalculateScalar(uint adler, ReadOnlySpan buffer) + { + uint s1 = adler & 0xFFFF; + uint s2 = (adler >> 16) & 0xFFFF; + uint k; + + ref byte bufferRef = ref MemoryMarshal.GetReference(buffer); + uint length = (uint)buffer.Length; + int index = 0; + + while (length > 0) + { + k = length < NMAX ? length : NMAX; + length -= k; + + while (k >= 16) + { + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + k -= 16; + } + + if (k != 0) + { + do + { + s1 += Unsafe.Add(ref bufferRef, index++); + s2 += s1; + } + while (--k != 0); + } + + s1 %= BASE; + s2 %= BASE; + } + + return (s2 << 16) | s1; + } + } +} \ No newline at end of file diff --git a/src/SharpCompress/Compressors/Deflate/DeflateManager.cs b/src/SharpCompress/Compressors/Deflate/DeflateManager.cs index c182d5e7..0abc4564 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateManager.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateManager.cs @@ -70,6 +70,8 @@ using System; +using SharpCompress.Algorithms; + namespace SharpCompress.Compressors.Deflate { internal sealed partial class DeflateManager @@ -1685,7 +1687,7 @@ namespace SharpCompress.Compressors.Deflate Rfc1950BytesEmitted = false; status = (WantRfc1950HeaderBytes) ? INIT_STATE : BUSY_STATE; - _codec._Adler32 = Adler.Adler32(0, null, 0, 0); + _codec._adler32 = 1; last_flush = (int)FlushType.None; @@ -1763,7 +1765,7 @@ namespace SharpCompress.Compressors.Deflate throw new ZlibException("Stream error."); } - _codec._Adler32 = Adler.Adler32(_codec._Adler32, dictionary, 0, dictionary.Length); + _codec._adler32 = Adler32.Calculate(_codec._adler32, dictionary); if (length < MIN_MATCH) { @@ -1850,12 +1852,12 @@ namespace SharpCompress.Compressors.Deflate ////putShortMSB((int)(SharedUtils.URShift(_codec._Adler32, 16))); //putShortMSB((int)((UInt64)_codec._Adler32 >> 16)); //putShortMSB((int)(_codec._Adler32 & 0xffff)); - pending[pendingCount++] = (byte)((_codec._Adler32 & 0xFF000000) >> 24); - pending[pendingCount++] = (byte)((_codec._Adler32 & 0x00FF0000) >> 16); - pending[pendingCount++] = (byte)((_codec._Adler32 & 0x0000FF00) >> 8); - pending[pendingCount++] = (byte)(_codec._Adler32 & 0x000000FF); + pending[pendingCount++] = (byte)((_codec._adler32 & 0xFF000000) >> 24); + pending[pendingCount++] = (byte)((_codec._adler32 & 0x00FF0000) >> 16); + pending[pendingCount++] = (byte)((_codec._adler32 & 0x0000FF00) >> 8); + pending[pendingCount++] = (byte)(_codec._adler32 & 0x000000FF); } - _codec._Adler32 = Adler.Adler32(0, null, 0, 0); + _codec._adler32 = 1; } // Flush as much pending output as possible @@ -1968,10 +1970,10 @@ namespace SharpCompress.Compressors.Deflate } // Write the zlib trailer (adler32) - pending[pendingCount++] = (byte)((_codec._Adler32 & 0xFF000000) >> 24); - pending[pendingCount++] = (byte)((_codec._Adler32 & 0x00FF0000) >> 16); - pending[pendingCount++] = (byte)((_codec._Adler32 & 0x0000FF00) >> 8); - pending[pendingCount++] = (byte)(_codec._Adler32 & 0x000000FF); + pending[pendingCount++] = (byte)((_codec._adler32 & 0xFF000000) >> 24); + pending[pendingCount++] = (byte)((_codec._adler32 & 0x00FF0000) >> 16); + pending[pendingCount++] = (byte)((_codec._adler32 & 0x0000FF00) >> 8); + pending[pendingCount++] = (byte)(_codec._adler32 & 0x000000FF); //putShortMSB((int)(SharedUtils.URShift(_codec._Adler32, 16))); //putShortMSB((int)(_codec._Adler32 & 0xffff)); diff --git a/src/SharpCompress/Compressors/Deflate/Inflate.cs b/src/SharpCompress/Compressors/Deflate/Inflate.cs index f9b66c0c..863f999f 100644 --- a/src/SharpCompress/Compressors/Deflate/Inflate.cs +++ b/src/SharpCompress/Compressors/Deflate/Inflate.cs @@ -65,6 +65,8 @@ using System; +using SharpCompress.Algorithms; + namespace SharpCompress.Compressors.Deflate { internal sealed class InflateBlocks @@ -118,7 +120,7 @@ namespace SharpCompress.Compressors.Deflate if (checkfn != null) { - _codec._Adler32 = check = Adler.Adler32(0, null, 0, 0); + _codec._adler32 = check = 1; } return oldCheck; } @@ -739,7 +741,7 @@ namespace SharpCompress.Compressors.Deflate // update check information if (checkfn != null) { - _codec._Adler32 = check = Adler.Adler32(check, window, readAt, nBytes); + _codec._adler32 = check = Adler32.Calculate(check, window.AsSpan(readAt, nBytes)); } // copy as far as end of window @@ -1764,7 +1766,7 @@ namespace SharpCompress.Compressors.Deflate _codec.AvailableBytesIn--; _codec.TotalBytesIn++; expectedCheck += (uint)(_codec.InputBuffer[_codec.NextIn++] & 0x000000ff); - _codec._Adler32 = expectedCheck; + _codec._adler32 = expectedCheck; mode = InflateManagerMode.DICT0; return ZlibConstants.Z_NEED_DICT; @@ -1879,12 +1881,12 @@ namespace SharpCompress.Compressors.Deflate throw new ZlibException("Stream error."); } - if (Adler.Adler32(1, dictionary, 0, dictionary.Length) != _codec._Adler32) + if (Adler32.Calculate(1, dictionary) != _codec._adler32) { return ZlibConstants.Z_DATA_ERROR; } - _codec._Adler32 = Adler.Adler32(0, null, 0, 0); + _codec._adler32 = 1; if (length >= (1 << wbits)) { diff --git a/src/SharpCompress/Compressors/Deflate/Zlib.cs b/src/SharpCompress/Compressors/Deflate/Zlib.cs index d33cbcc9..6133feb4 100644 --- a/src/SharpCompress/Compressors/Deflate/Zlib.cs +++ b/src/SharpCompress/Compressors/Deflate/Zlib.cs @@ -405,88 +405,4 @@ namespace SharpCompress.Compressors.Deflate BitLengths = new StaticTree(null, extra_blbits, 0, InternalConstants.BL_CODES, InternalConstants.MAX_BL_BITS); } } - - /// - /// Computes an Adler-32 checksum. - /// - /// - /// The Adler checksum is similar to a CRC checksum, but faster to compute, though less - /// reliable. It is used in producing RFC1950 compressed streams. The Adler checksum - /// is a required part of the "ZLIB" standard. Applications will almost never need to - /// use this class directly. - /// - internal sealed class Adler - { - // largest prime smaller than 65536 - private static readonly uint BASE = 65521U; - - // NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 - private static readonly int NMAX = 5552; - - internal static uint Adler32(uint adler, byte[]? buf, int index, int len) - { - if (buf is null) - { - return 1; - } - - uint s1 = adler & 0xffffU; - uint s2 = (adler >> 16) & 0xffffU; - - while (len > 0) - { - int k = len < NMAX ? len : NMAX; - len -= k; - while (k >= 16) - { - //s1 += (buf[index++] & 0xff); s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - s1 += buf[index++]; - s2 += s1; - k -= 16; - } - if (k != 0) - { - do - { - s1 += buf[index++]; - s2 += s1; - } - while (--k != 0); - } - s1 %= BASE; - s2 %= BASE; - } - return (s2 << 16) | s1; - } - } } \ No newline at end of file diff --git a/src/SharpCompress/Compressors/Deflate/ZlibCodec.cs b/src/SharpCompress/Compressors/Deflate/ZlibCodec.cs index e7e2ecb9..8b995018 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibCodec.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibCodec.cs @@ -137,7 +137,7 @@ namespace SharpCompress.Compressors.Deflate internal DeflateManager dstate; internal InflateManager istate; - internal uint _Adler32; + internal uint _adler32; /// /// The compression level to use in this codec. Useful only in compression mode. @@ -173,7 +173,7 @@ namespace SharpCompress.Compressors.Deflate /// /// The Adler32 checksum on the data transferred through the codec so far. You probably don't need to look at this. /// - public int Adler32 => (int)_Adler32; + public int Adler32 => (int)_adler32; /// /// Create a ZlibCodec. @@ -737,7 +737,7 @@ namespace SharpCompress.Compressors.Deflate if (dstate.WantRfc1950HeaderBytes) { - _Adler32 = Adler.Adler32(_Adler32, InputBuffer, NextIn, len); + _adler32 = Algorithms.Adler32.Calculate(_adler32, InputBuffer.AsSpan(NextIn, len)); } Array.Copy(InputBuffer, NextIn, buf, start, len); NextIn += len;