starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-04-05 21:51:09 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Merge pull request #1266 from coderb/master

Browse Source 
Fix three BLAKE2sp correctness bugs and eliminate allocations in hot path
This commit is contained in:
Adam Hathcock
2026-03-28 10:03:58 +00:00
committed by GitHub
parent fefeed0601 512f101d26
commit ea6946c48d
2 changed files with 135 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.Async.cs
View File

@@ -1,5 +1,4 @@
using System; using System;
using System.IO;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using SharpCompress.Common; using SharpCompress.Common;
@@ -32,12 +31,12 @@ internal partial class RarBLAKE2spStream : RarStream
.ConfigureAwait(false); .ConfigureAwait(false);
if (result != 0) if (result != 0)
{ {
Update(_blake2sp, new ReadOnlySpan<byte>(buffer, offset, result), result); Update(_blake2sp!, new ReadOnlySpan<byte>(buffer, offset, result));
} }
else else
{ {
_hash = Final(_blake2sp); EnsureHash();
if (!disableCRCCheck && !(GetCrc().SequenceEqual(readStream.CurrentCrc)) && count != 0) if (!disableCRCCheck && !GetCrc().SequenceEqual(readStream.CurrentCrc) && count != 0)
{ {
// NOTE: we use the last FileHeader in a multipart volume to check CRC // NOTE: we use the last FileHeader in a multipart volume to check CRC
throw new InvalidFormatException("file crc mismatch"); throw new InvalidFormatException("file crc mismatch");
@@ -56,14 +55,14 @@ internal partial class RarBLAKE2spStream : RarStream
var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
if (result != 0) if (result != 0)
{ {
Update(_blake2sp, buffer.Span.Slice(0, result), result); Update(_blake2sp!, buffer.Span.Slice(0, result));
} }
else else
{ {
_hash = Final(_blake2sp); EnsureHash();
if ( if (
!disableCRCCheck !disableCRCCheck
&& !(GetCrc().SequenceEqual(readStream.CurrentCrc)) && !GetCrc().SequenceEqual(readStream.CurrentCrc)
&& buffer.Length != 0 && buffer.Length != 0
) )
{ {

241
src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs
View File

@@ -1,8 +1,5 @@
using System; using System;
using System.IO; using System.Runtime.InteropServices;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common; using SharpCompress.Common;
using SharpCompress.Common.Rar.Headers; using SharpCompress.Common.Rar.Headers;
@@ -13,14 +10,14 @@ internal partial class RarBLAKE2spStream : RarStream
private readonly MultiVolumeReadOnlyStreamBase readStream; private readonly MultiVolumeReadOnlyStreamBase readStream;
private readonly bool disableCRCCheck; private readonly bool disableCRCCheck;
const uint BLAKE2S_NUM_ROUNDS = 10; private const int BLAKE2S_NUM_ROUNDS = 10;
const uint BLAKE2S_FINAL_FLAG = (~(uint)0); private const uint BLAKE2S_FINAL_FLAG = ~(uint)0;
const int BLAKE2S_BLOCK_SIZE = 64; private const int BLAKE2S_BLOCK_SIZE = 64;
const int BLAKE2S_DIGEST_SIZE = 32; private const int BLAKE2S_DIGEST_SIZE = 32;
const int BLAKE2SP_PARALLEL_DEGREE = 8; private const int BLAKE2SP_PARALLEL_DEGREE = 8;
const uint BLAKE2S_INIT_IV_SIZE = 8; private const int BLAKE2S_INIT_IV_SIZE = 8;
static readonly UInt32[] k_BLAKE2S_IV = private static readonly uint[] k_BLAKE2S_IV =
{ {
0x6A09E667U, 0x6A09E667U,
0xBB67AE85U, 0xBB67AE85U,
@@ -32,7 +29,7 @@ internal partial class RarBLAKE2spStream : RarStream
0x5BE0CD19U, 0x5BE0CD19U,
}; };
static readonly byte[][] k_BLAKE2S_Sigma = private static readonly byte[][] k_BLAKE2S_Sigma =
{ {
new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
new byte[] { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, new byte[] { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 },
@@ -46,14 +43,14 @@ internal partial class RarBLAKE2spStream : RarStream
new byte[] { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, new byte[] { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
}; };
internal class BLAKE2S private sealed class BLAKE2S
{ {
internal UInt32[] h; internal readonly uint[] h;
internal UInt32[] t; internal readonly uint[] t;
internal UInt32[] f; internal readonly uint[] f;
internal byte[] b; internal readonly byte[] b;
internal int bufferPosition; internal int bufferPosition;
internal UInt32 lastNodeFlag; internal uint lastNodeFlag;
public BLAKE2S() public BLAKE2S()
{ {
@@ -64,9 +61,9 @@ internal partial class RarBLAKE2spStream : RarStream
} }
}; };
internal class BLAKE2SP private sealed class BLAKE2SP
{ {
internal BLAKE2S[] S; internal readonly BLAKE2S[] S;
internal int bufferPosition; internal int bufferPosition;
public BLAKE2SP() public BLAKE2SP()
@@ -79,9 +76,8 @@ internal partial class RarBLAKE2spStream : RarStream
} }
}; };
BLAKE2SP _blake2sp; private BLAKE2SP? _blake2sp;
private byte[]? _hash;
byte[] _hash = [];
private RarBLAKE2spStream( private RarBLAKE2spStream(
IRarUnpack unpack, IRarUnpack unpack,
@@ -92,10 +88,9 @@ internal partial class RarBLAKE2spStream : RarStream
{ {
this.readStream = readStream; this.readStream = readStream;
// TODO: rar uses a modified hash xor'ed with encryption key?
disableCRCCheck = fileHeader.IsEncrypted; disableCRCCheck = fileHeader.IsEncrypted;
_hash = fileHeader.FileCrc.NotNull(); this._blake2sp = CreateBlake2sp();
_blake2sp = new BLAKE2SP();
ResetCrc();
} }
public static RarBLAKE2spStream Create( public static RarBLAKE2spStream Create(
@@ -111,19 +106,15 @@ internal partial class RarBLAKE2spStream : RarStream
// Async methods moved to RarBLAKE2spStream.Async.cs // Async methods moved to RarBLAKE2spStream.Async.cs
protected override void Dispose(bool disposing) public byte[] GetCrc() =>
{ this._hash
base.Dispose(disposing); ?? throw new InvalidOperationException(
} "hash not computed, has the stream been fully drained?"
);
public byte[] GetCrc() => _hash; private static void ResetCrc(BLAKE2S hash)
internal void ResetCrc(BLAKE2S hash)
{ {
for (UInt32 j = 0; j < BLAKE2S_INIT_IV_SIZE; j++) k_BLAKE2S_IV.AsSpan().CopyTo(hash.h);
{
hash.h[j] = k_BLAKE2S_IV[j];
}
hash.t[0] = 0; hash.t[0] = 0;
hash.t[1] = 0; hash.t[1] = 0;
hash.f[0] = 0; hash.f[0] = 0;
@@ -132,14 +123,14 @@ internal partial class RarBLAKE2spStream : RarStream
hash.lastNodeFlag = 0; hash.lastNodeFlag = 0;
} }
internal void G( private static void G(
ref UInt32[] m, Span<uint> m,
ref byte[] sigma, byte[] sigma,
int i, int i,
ref UInt32 a, ref uint a,
ref UInt32 b, ref uint b,
ref UInt32 c, ref uint c,
ref UInt32 d ref uint d
) )
{ {
a += b + m[sigma[2 * i]]; a += b + m[sigma[2 * i]];
@@ -157,16 +148,22 @@ internal partial class RarBLAKE2spStream : RarStream
b = (b >> 7) | (b << 25); b = (b >> 7) | (b << 25);
} }
internal void Compress(BLAKE2S hash) private static void Compress(BLAKE2S hash)
{ {
var m = new UInt32[16]; Span<uint> m = stackalloc uint[16];
var v = new UInt32[16]; if (BitConverter.IsLittleEndian)
for (var i = 0; i < 16; i++)
{ {
m[i] = BitConverter.ToUInt32(hash.b, i * 4); MemoryMarshal.Cast<byte, uint>(hash.b).CopyTo(m);
}
else
{
for (var i = 0; i < 16; i++)
{
m[i] = BitConverter.ToUInt32(hash.b, i * 4);
}
} }
Span<uint> v = stackalloc uint[16];
for (var i = 0; i < 8; i++) for (var i = 0; i < 8; i++)
{ {
v[i] = hash.h[i]; v[i] = hash.h[i];
@@ -184,16 +181,15 @@ internal partial class RarBLAKE2spStream : RarStream
for (var r = 0; r < BLAKE2S_NUM_ROUNDS; r++) for (var r = 0; r < BLAKE2S_NUM_ROUNDS; r++)
{ {
ref byte[] sigma = ref k_BLAKE2S_Sigma[r]; var sigma = k_BLAKE2S_Sigma[r];
G(m, sigma, 0, ref v[0], ref v[4], ref v[8], ref v[12]);
G(ref m, ref sigma, 0, ref v[0], ref v[4], ref v[8], ref v[12]); G(m, sigma, 1, ref v[1], ref v[5], ref v[9], ref v[13]);
G(ref m, ref sigma, 1, ref v[1], ref v[5], ref v[9], ref v[13]); G(m, sigma, 2, ref v[2], ref v[6], ref v[10], ref v[14]);
G(ref m, ref sigma, 2, ref v[2], ref v[6], ref v[10], ref v[14]); G(m, sigma, 3, ref v[3], ref v[7], ref v[11], ref v[15]);
G(ref m, ref sigma, 3, ref v[3], ref v[7], ref v[11], ref v[15]); G(m, sigma, 4, ref v[0], ref v[5], ref v[10], ref v[15]);
G(ref m, ref sigma, 4, ref v[0], ref v[5], ref v[10], ref v[15]); G(m, sigma, 5, ref v[1], ref v[6], ref v[11], ref v[12]);
G(ref m, ref sigma, 5, ref v[1], ref v[6], ref v[11], ref v[12]); G(m, sigma, 6, ref v[2], ref v[7], ref v[8], ref v[13]);
G(ref m, ref sigma, 6, ref v[2], ref v[7], ref v[8], ref v[13]); G(m, sigma, 7, ref v[3], ref v[4], ref v[9], ref v[14]);
G(ref m, ref sigma, 7, ref v[3], ref v[4], ref v[9], ref v[14]);
} }
for (var i = 0; i < 8; i++) for (var i = 0; i < 8; i++)
@@ -202,103 +198,124 @@ internal partial class RarBLAKE2spStream : RarStream
} }
} }
internal void Update(BLAKE2S hash, ReadOnlySpan<byte> data, int size) private static void Update(BLAKE2S hash, ReadOnlySpan<byte> data)
{ {
var i = 0; while (data.Length != 0)
while (size != 0)
{ {
var pos = hash.bufferPosition; var pos = hash.bufferPosition;
var reminder = BLAKE2S_BLOCK_SIZE - pos; var chunkSize = BLAKE2S_BLOCK_SIZE - pos;
if (data.Length <= chunkSize)
if (size <= reminder)
{ {
data.Slice(i, size).CopyTo(new Span<byte>(hash.b, pos, size)); data.CopyTo(hash.b.AsSpan(pos));
hash.bufferPosition += size; hash.bufferPosition += data.Length;
return; return;
} }
data.Slice(i, reminder).CopyTo(new Span<byte>(hash.b, pos, reminder)); data.Slice(0, chunkSize).CopyTo(hash.b.AsSpan(pos));
hash.t[0] += BLAKE2S_BLOCK_SIZE; hash.t[0] += BLAKE2S_BLOCK_SIZE;
hash.t[1] += hash.t[0] < BLAKE2S_BLOCK_SIZE ? 1U : 0U; hash.t[1] += hash.t[0] < BLAKE2S_BLOCK_SIZE ? 1U : 0U;
Compress(hash); Compress(hash);
hash.bufferPosition = 0; hash.bufferPosition = 0;
i += reminder; data = data.Slice(chunkSize);
size -= reminder;
} }
} }
internal byte[] Final(BLAKE2S hash) private static void Final(BLAKE2S hash, Span<byte> output)
{ {
hash.t[0] += (uint)hash.bufferPosition; hash.t[0] += (uint)hash.bufferPosition;
hash.t[1] += hash.t[0] < hash.bufferPosition ? 1U : 0U; hash.t[1] += hash.t[0] < hash.bufferPosition ? 1U : 0U;
hash.f[0] = BLAKE2S_FINAL_FLAG; hash.f[0] = BLAKE2S_FINAL_FLAG;
hash.f[1] = hash.lastNodeFlag; hash.f[1] = hash.lastNodeFlag;
Array.Clear(hash.b, hash.bufferPosition, BLAKE2S_BLOCK_SIZE - hash.bufferPosition); hash.b.AsSpan(hash.bufferPosition).Clear();
Compress(hash); Compress(hash);
var mem = new MemoryStream(); if (BitConverter.IsLittleEndian)
for (var i = 0; i < 8; i++)
{ {
mem.Write(BitConverter.GetBytes(hash.h[i]), 0, 4); MemoryMarshal.Cast<uint, byte>(hash.h).CopyTo(output);
}
else
{
for (var i = 0; i < 8; i++)
{
var v = hash.h[i];
output[i * 4] = (byte)v;
output[i * 4 + 1] = (byte)(v >> 8);
output[i * 4 + 2] = (byte)(v >> 16);
output[i * 4 + 3] = (byte)(v >> 24);
}
} }
return mem.ToArray();
} }
public void ResetCrc() private static BLAKE2SP CreateBlake2sp()
{ {
_blake2sp.bufferPosition = 0; var blake2sp = new BLAKE2SP();
for (UInt32 i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) for (var i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++)
{ {
_blake2sp.S[i].bufferPosition = 0; var blake2S = blake2sp.S[i];
ResetCrc(_blake2sp.S[i]); ResetCrc(blake2S);
_blake2sp.S[i].h[0] ^= (BLAKE2S_DIGEST_SIZE | BLAKE2SP_PARALLEL_DEGREE << 16 | 2 << 24);
_blake2sp.S[i].h[2] ^= i; var h = blake2S.h;
_blake2sp.S[i].h[3] ^= (BLAKE2S_DIGEST_SIZE << 24); // word[0]: digest_length | (fanout<<16) | (depth<<24)
h[0] ^= BLAKE2S_DIGEST_SIZE | (BLAKE2SP_PARALLEL_DEGREE << 16) | (2 << 24);
// word[2]: node_offset = leaf index
h[2] ^= (uint)i;
// word[3]: inner_length in bits 24-31
h[3] ^= BLAKE2S_DIGEST_SIZE << 24;
} }
_blake2sp.S[BLAKE2SP_PARALLEL_DEGREE - 1].lastNodeFlag = BLAKE2S_FINAL_FLAG; blake2sp.S[BLAKE2SP_PARALLEL_DEGREE - 1].lastNodeFlag = BLAKE2S_FINAL_FLAG;
return blake2sp;
} }
internal void Update(BLAKE2SP hash, ReadOnlySpan<byte> data, int size) private static void Update(BLAKE2SP hash, ReadOnlySpan<byte> data)
{ {
var i = 0;
var pos = hash.bufferPosition; var pos = hash.bufferPosition;
while (size != 0) while (data.Length != 0)
{ {
var index = pos / BLAKE2S_BLOCK_SIZE; var index = pos / BLAKE2S_BLOCK_SIZE;
var reminder = BLAKE2S_BLOCK_SIZE - (pos & (BLAKE2S_BLOCK_SIZE - 1)); var chunkSize = BLAKE2S_BLOCK_SIZE - (pos & (BLAKE2S_BLOCK_SIZE - 1));
if (reminder > size) if (chunkSize > data.Length)
{ {
reminder = size; chunkSize = data.Length;
} }
// Update(hash.S[index], data, size); Update(hash.S[index], data.Slice(0, chunkSize));
Update(hash.S[index], data.Slice(i, reminder), reminder); data = data.Slice(chunkSize);
size -= reminder; pos = (pos + chunkSize) & (BLAKE2S_BLOCK_SIZE * BLAKE2SP_PARALLEL_DEGREE - 1);
i += reminder;
pos += reminder;
pos &= (BLAKE2S_BLOCK_SIZE * (BLAKE2SP_PARALLEL_DEGREE - 1));
} }
hash.bufferPosition = pos; hash.bufferPosition = pos;
} }
internal byte[] Final(BLAKE2SP hash) private static byte[] Final(BLAKE2SP blake2sp)
{ {
var h = new BLAKE2S(); var blake2s = new BLAKE2S();
ResetCrc(blake2s);
ResetCrc(h); var h = blake2s.h;
h.h[0] ^= (BLAKE2S_DIGEST_SIZE | BLAKE2SP_PARALLEL_DEGREE << 16 | 2 << 24); // word[0]: digest_length | (fanout<<16) | (depth<<24) — same as leaves
h.h[3] ^= (1 << 16 | BLAKE2S_DIGEST_SIZE << 24); h[0] ^= BLAKE2S_DIGEST_SIZE | (BLAKE2SP_PARALLEL_DEGREE << 16) | (2 << 24);
h.lastNodeFlag = BLAKE2S_FINAL_FLAG; // word[3]: node_depth=1 (bits 16-23), inner_length=32 (bits 24-31)
h[3] ^= (1 << 16) | (BLAKE2S_DIGEST_SIZE << 24);
blake2s.lastNodeFlag = BLAKE2S_FINAL_FLAG;
Span<byte> digest = stackalloc byte[BLAKE2S_DIGEST_SIZE];
for (var i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) for (var i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++)
{ {
var digest = Final(_blake2sp.S[i]); Final(blake2sp.S[i], digest);
Update(h, digest, BLAKE2S_DIGEST_SIZE); Update(blake2s, digest);
} }
return Final(h); Final(blake2s, digest);
return digest.ToArray();
}
private void EnsureHash()
{
if (this._hash == null)
{
this._hash = Final(this._blake2sp!);
// prevent incorrect usage past hash finality by failing fast
this._blake2sp = null;
}
} }
public override int Read(byte[] buffer, int offset, int count) public override int Read(byte[] buffer, int offset, int count)
@@ -306,12 +323,12 @@ internal partial class RarBLAKE2spStream : RarStream
var result = base.Read(buffer, offset, count); var result = base.Read(buffer, offset, count);
if (result != 0) if (result != 0)
{ {
Update(_blake2sp, new ReadOnlySpan<byte>(buffer, offset, result), result); Update(this._blake2sp!, new ReadOnlySpan<byte>(buffer, offset, result));
} }
else else
{ {
_hash = Final(_blake2sp); EnsureHash();
if (!disableCRCCheck && !(GetCrc().SequenceEqual(readStream.CurrentCrc)) && count != 0) if (!disableCRCCheck && !GetCrc().SequenceEqual(readStream.CurrentCrc) && count != 0)
{ {
// NOTE: we use the last FileHeader in a multipart volume to check CRC // NOTE: we use the last FileHeader in a multipart volume to check CRC
throw new InvalidFormatException("file crc mismatch"); throw new InvalidFormatException("file crc mismatch");