starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-22 06:54:40 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Improve CRC32 by using the slice-by-16 optimization

Browse Source
This commit is contained in:
Julian Xhokaxhiu
2026-08-02 23:11:05 +02:00
parent 187055e673
commit 5f340db48d
1 changed files with 64 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
src/SharpCompress/Crypto/Crc32Stream.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Buffers.Binary;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@@ -119,6 +120,14 @@ public sealed class Crc32Stream : Stream
public static uint Compute(uint polynomial, uint seed, ReadOnlySpan<byte> buffer) =>
~CalculateCrc(InitializeTable(polynomial), seed, buffer);
// Number of "slice-by-N" tables generated per polynomial (see InitializeTable). 16 gives a
// good throughput/table-size tradeoff (16 * 256 * 4 bytes = 16KB per polynomial, generated
// once and cached for the default polynomial) - a well-known technique (zlib-ng, Chromium,
// etc.) for speeding up table-driven CRC32 by processing 16 bytes/iteration through 16
// independent (parallelizable) table lookups instead of 1 byte/iteration through a single
// lookup on the CPU's dependency chain.
private const int SliceTableCount = 16;
internal static uint[] InitializeTable(uint polynomial)
{
if (polynomial == DEFAULT_POLYNOMIAL && _defaultTable != null)
@@ -126,38 +135,80 @@ public sealed class Crc32Stream : Stream
return _defaultTable;
}
var createTable = new uint[256];
// table[0..255] is the ordinary byte-wise CRC table (unchanged layout, so existing
// single-byte callers - CalculateCrc(table, crc, byte) - keep working unmodified).
// table[256*k .. 256*k+255] for k=1..SliceTableCount-1 are the derived "slice" tables:
// table[k][i] = (table[k-1][i] >> 8) ^ table[0][table[k-1][i] & 0xFF], i.e. each table
// folds in the effect of a byte that is k positions further back in the stream than
// the byte table[0] would have processed - letting CalculateCrc(span) below combine 4
// bytes' worth of independent lookups per 32-bit word with no cross-lookup dependency.
var table = new uint[256 * SliceTableCount];
for (var i = 0; i < 256; i++)
{
var entry = (uint)i;
for (var j = 0; j < 8; j++)
{
if ((entry & 1) == 1)
{
entry = (entry >> 1) ^ polynomial;
}
else
{
entry >>= 1;
}
entry = (entry & 1) == 1 ? (entry >> 1) ^ polynomial : entry >> 1;
}
table[i] = entry;
}
createTable[i] = entry;
for (var k = 1; k < SliceTableCount; k++)
{
var prevBase = (k - 1) * 256;
var curBase = k * 256;
for (var i = 0; i < 256; i++)
{
var prev = table[prevBase + i];
table[curBase + i] = (prev >> 8) ^ table[prev & 0xFF];
}
}
if (polynomial == DEFAULT_POLYNOMIAL)
{
_defaultTable = createTable;
_defaultTable = table;
}
return createTable;
return table;
}
internal static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan<byte> buffer)
{
unchecked
{
for (var i = 0; i < buffer.Length; i++)
var i = 0;
if (table.Length >= 256 * SliceTableCount)
{
while (buffer.Length - i >= 16)
{
var word0 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i)) ^ crc;
var word1 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 4));
var word2 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 8));
var word3 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 12));
crc =
table[15 * 256 + (byte)word0]
^ table[14 * 256 + (byte)(word0 >> 8)]
^ table[13 * 256 + (byte)(word0 >> 16)]
^ table[12 * 256 + (word0 >> 24)]
^ table[11 * 256 + (byte)word1]
^ table[10 * 256 + (byte)(word1 >> 8)]
^ table[9 * 256 + (byte)(word1 >> 16)]
^ table[8 * 256 + (word1 >> 24)]
^ table[7 * 256 + (byte)word2]
^ table[6 * 256 + (byte)(word2 >> 8)]
^ table[5 * 256 + (byte)(word2 >> 16)]
^ table[4 * 256 + (word2 >> 24)]
^ table[3 * 256 + (byte)word3]
^ table[2 * 256 + (byte)(word3 >> 8)]
^ table[1 * 256 + (byte)(word3 >> 16)]
^ table[word3 >> 24];
i += 16;
}
}
for (; i < buffer.Length; i++)
{
crc = CalculateCrc(table, crc, buffer[i]);
}