From e61a335d54fd473a2358cc85a779efd476070d55 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 20 Oct 2026 10:10:29 +0100 Subject: [PATCH 1/2] Reduce allocations in RAR5 password key derivation Reuse keyed HMAC state and digest buffers on modern .NET targets while preserving RAR5's three PBKDF2 snapshots. Add independent derivation checks and an allocation regression test. Profiling on macOS ARM64 with .NET 10 reduced allocations by 99.96%, but showed slower throughput; Alpine-specific timing remains unverified. Full .NET 10 test suite passes. --- src/SharpCompress/Common/Rar/CryptKey5.cs | 21 +++++-- .../SharpCompress.Test/Rar/CryptKey5Tests.cs | 58 +++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/SharpCompress/Common/Rar/CryptKey5.cs b/src/SharpCompress/Common/Rar/CryptKey5.cs index ee5f0c3b..c68fbc8a 100644 --- a/src/SharpCompress/Common/Rar/CryptKey5.cs +++ b/src/SharpCompress/Common/Rar/CryptKey5.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; @@ -35,14 +36,21 @@ internal class CryptKey5 : ICryptKey { var passwordBytes = Encoding.UTF8.GetBytes(password); salt = salt.Concat(new byte[] { 0, 0, 0, 1 }).ToArray(); -#if LEGACY_DOTNET + // Reuse keyed HMAC state: recreating it for every iteration is especially + // expensive on some platforms. Modern targets also reuse digest buffers. using var hmac = new HMACSHA256(passwordBytes); +#if LEGACY_DOTNET var block = hmac.ComputeHash(salt); -#else - var block = HMACSHA256.HashData(passwordBytes, salt); -#endif var finalHash = (byte[])block.Clone(); +#else + Span block = stackalloc byte[SHA256_DIGEST_SIZE]; + Span nextBlock = stackalloc byte[SHA256_DIGEST_SIZE]; + Span finalHash = stackalloc byte[SHA256_DIGEST_SIZE]; + hmac.TryComputeHash(salt, block, out _); + block.CopyTo(finalHash); +#endif + // RAR5 takes snapshots of the same PBKDF2 chain at N, N+16, and N+32. var loop = new int[] { iterations, 17, 17 }; var res = new List { }; @@ -53,7 +61,8 @@ internal class CryptKey5 : ICryptKey #if LEGACY_DOTNET block = hmac.ComputeHash(block); #else - block = HMACSHA256.HashData(passwordBytes, block); + hmac.TryComputeHash(block, nextBlock, out _); + nextBlock.CopyTo(block); #endif for (var j = 0; j < finalHash.Length; j++) { @@ -61,7 +70,7 @@ internal class CryptKey5 : ICryptKey } } - res.Add((byte[])finalHash.Clone()); + res.Add(finalHash.ToArray()); } return res; diff --git a/tests/SharpCompress.Test/Rar/CryptKey5Tests.cs b/tests/SharpCompress.Test/Rar/CryptKey5Tests.cs index 45665b2c..648c156a 100644 --- a/tests/SharpCompress.Test/Rar/CryptKey5Tests.cs +++ b/tests/SharpCompress.Test/Rar/CryptKey5Tests.cs @@ -1,5 +1,8 @@ +using System; using System.IO; using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Threading.Tasks; using SharpCompress.Common.Rar; using SharpCompress.Common.Rar.Headers; @@ -11,6 +14,61 @@ namespace SharpCompress.Test.Rar; public class CryptKey5Tests : TestBase { + [Theory] + [InlineData("", 1)] + [InlineData("password", 2)] + [InlineData("password", 4096)] + [InlineData("pässwörd🔐", 32)] + [InlineData( + "a password longer than the SHA256 block size of sixty-four bytes to exercise HMAC key hashing", + 128 + )] + public void Rar5_DerivedKeys_MatchPbkdf2_Snapshots(string password, int iterations) + { + var salt = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + var originalSalt = (byte[])salt.Clone(); + var keys = CryptKey5.GenerateRarPBKDF2Key(password, salt, iterations); + + Assert.Equal(3, keys.Count); + for (var i = 0; i < keys.Count; i++) + { +#if LEGACY_DOTNET + using var reference = new Rfc2898DeriveBytes( + Encoding.UTF8.GetBytes(password), + salt, + iterations + i * 16, + HashAlgorithmName.SHA256 + ); + Assert.Equal(reference.GetBytes(32), keys[i]); +#else + var expected = Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(password), + salt, + iterations + i * 16, + HashAlgorithmName.SHA256, + 32 + ); + Assert.Equal(expected, keys[i]); +#endif + } + Assert.Equal(originalSalt, salt); + } + +#if !LEGACY_DOTNET + [Fact] + public void Rar5_DerivedKeys_DoNotAllocate_PerIteration() + { + var salt = new byte[16]; + CryptKey5.GenerateRarPBKDF2Key("password", salt, 4096); + var before = GC.GetAllocatedBytesForCurrentThread(); + CryptKey5.GenerateRarPBKDF2Key("password", salt, 4096); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + // Allow fixed setup costs, but not a digest array for each iteration. + Assert.True(allocated < 16 * 1024, $"Allocated {allocated} bytes."); + } +#endif + [Theory] [InlineData(0)] [InlineData(24)] From f63cd81a65441ff13da586758ff337d77e227933 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 20 Oct 2026 10:43:28 +0100 Subject: [PATCH 2/2] Reduce allocations in encrypted RAR reading Reuse bounded buffers and transform AES blocks in place. Remove per-block arrays and plaintext queues, avoid per-byte header allocations, and dispose owned crypto transforms. Add encrypted RAR benchmarks, regression tests, and before/after profiling results. Full .NET 10 suite passes (2,446 tests); all library targets build and net48 tests compile. --- .../Common/Rar/AsyncRarCryptoBinaryReader.cs | 68 ++++--- .../Rar/Headers/RarHeaderFactory.Async.cs | 2 + .../Common/Rar/Headers/RarHeaderFactory.cs | 2 + .../Common/Rar/RarCryptoBinaryReader.cs | 58 +++--- .../Common/Rar/RarCryptoWrapper.cs | 166 +++++++++++------- src/SharpCompress/Crypto/BlockTransformer.cs | 13 +- .../Benchmarks/RarEncryptedBenchmarks.cs | 104 +++++++++++ .../rar-crypto-profiling.md | 65 +++++++ .../Rar/RarCryptoWrapperTests.cs | 165 ++++++++++++++++- 9 files changed, 511 insertions(+), 132 deletions(-) create mode 100644 tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs create mode 100644 tests/SharpCompress.Performance/rar-crypto-profiling.md diff --git a/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs b/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs index c3f5d005..c79896a3 100644 --- a/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs +++ b/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs @@ -1,17 +1,18 @@ using System; -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; using SharpCompress.Crypto; +using SharpCompress.IO; namespace SharpCompress.Common.Rar; internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader { private BlockTransformer _rijndael = default!; - private readonly Queue _data = new(); + private readonly byte[] _block = new byte[16]; + private int _blockOffset = 16; private long _readCount; private AsyncRarCryptoBinaryReader(Stream stream) @@ -50,8 +51,26 @@ internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader CancellationToken cancellationToken = default ) { - var bytes = await ReadAndDecryptBytesAsync(1, cancellationToken).ConfigureAwait(false); - return bytes[0]; + cancellationToken.ThrowIfCancellationRequested(); + if (_blockOffset == _block.Length) + { + try + { + await BaseStream + .ReadExactAsync(_block, 0, _block.Length, cancellationToken) + .ConfigureAwait(false); + } + catch (IncompleteArchiveException e) + { + throw new InvalidFormatException("Unexpected end of encrypted stream", e); + } + _rijndael.ProcessBlocks(_block, _block.Length); + _blockOffset = 0; + } + var value = _block[_blockOffset++]; + UpdateCrc(value); + _readCount++; + return value; } private ValueTask ReadBytesAsyncBase(int count) => base.ReadBytesAsync(count); @@ -69,43 +88,36 @@ internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader CancellationToken cancellationToken ) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) - { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - for (var i = 0; i < alignedSize / 16; i++) - { - var cipherText = await ReadBytesNoCrcAsync(16, cancellationToken) - .ConfigureAwait(false); - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } - } - } - var decryptedBytes = new byte[count]; for (var i = 0; i < count; i++) { - var b = _data.Dequeue(); - decryptedBytes[i] = b; - UpdateCrc(b); + decryptedBytes[i] = await ReadByteAsync(cancellationToken).ConfigureAwait(false); } - _readCount += count; return decryptedBytes; } - public void ClearQueue() => _data.Clear(); + public void ClearQueue() => _blockOffset = _block.Length; public void SkipQueue() { var position = BaseStream.Position; - BaseStream.Position = position + _data.Count; + BaseStream.Position = position + _block.Length - _blockOffset; ClearQueue(); } + + public override void Dispose() + { + _rijndael.Dispose(); + base.Dispose(); + } + +#if NET8_0_OR_GREATER + public override ValueTask DisposeAsync() + { + _rijndael.Dispose(); + return base.DisposeAsync(); + } +#endif } diff --git a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs index a6108b32..85f69db4 100644 --- a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs @@ -79,6 +79,8 @@ public partial class RarHeaderFactory } } + // Readers leave the underlying stream open but own their crypto transforms. + using var readerLifetime = reader; var header = await RarHeader .TryReadBaseAsync(reader, _isRar5, Options.ArchiveEncoding, cancellationToken) .ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs index 27ae9db6..4fb9c00b 100644 --- a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs +++ b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs @@ -98,6 +98,8 @@ public partial class RarHeaderFactory } } + // Readers leave the underlying stream open but own their crypto transforms. + using var readerLifetime = reader; var header = RarHeader.TryReadBase(reader, _isRar5, Options.ArchiveEncoding); if (header is null) { diff --git a/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs b/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs index 319cf4c5..60299057 100644 --- a/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs +++ b/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs @@ -1,14 +1,15 @@ -using System.Collections.Generic; using System.IO; using SharpCompress.Common.Rar.Headers; using SharpCompress.Crypto; +using SharpCompress.IO; namespace SharpCompress.Common.Rar; internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader { private BlockTransformer _rijndael = default!; - private readonly Queue _data = new(); + private readonly byte[] _block = new byte[16]; + private int _blockOffset = 16; private long _readCount; private RarCryptoBinaryReader(Stream stream) @@ -42,7 +43,22 @@ internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader public override void Mark() => _readCount = 0; - public override byte ReadByte() => ReadAndDecryptBytes(1)[0]; + public override byte ReadByte() + { + if (_blockOffset == _block.Length) + { + if (!BaseStream.ReadFully(_block)) + { + throw new InvalidFormatException("Unexpected end of encrypted stream"); + } + _rijndael.ProcessBlocks(_block, _block.Length); + _blockOffset = 0; + } + var value = _block[_blockOffset++]; + UpdateCrc(value); + _readCount++; + return value; + } public override byte[] ReadBytes(int count) => ReadAndDecryptBytes(count); @@ -50,43 +66,31 @@ internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader private byte[] ReadAndDecryptBytes(int count) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) - { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - for (var i = 0; i < alignedSize / 16; i++) - { - //long ax = System.currentTimeMillis(); - var cipherText = ReadBytesNoCrc(16); - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } - } - } - var decryptedBytes = new byte[count]; for (var i = 0; i < count; i++) { - var b = _data.Dequeue(); - decryptedBytes[i] = b; - UpdateCrc(b); + decryptedBytes[i] = ReadByte(); } - _readCount += count; return decryptedBytes; } - public void ClearQueue() => _data.Clear(); + public void ClearQueue() => _blockOffset = _block.Length; public void SkipQueue() { var position = BaseStream.Position; - BaseStream.Position = position + _data.Count; + BaseStream.Position = position + _block.Length - _blockOffset; ClearQueue(); } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _rijndael.Dispose(); + } + base.Dispose(disposing); + } } diff --git a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs index d5e7112b..55bc7425 100644 --- a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs +++ b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -11,8 +10,10 @@ namespace SharpCompress.Common.Rar; internal sealed class RarCryptoWrapper : Stream { private readonly Stream _actualStream; - private BlockTransformer _rijndael; - private readonly Queue _data = new(); + private readonly BlockTransformer _rijndael; + private byte[] _buffer = new byte[16]; + private int _offset; + private int _available; public RarCryptoWrapper(Stream actualStream, byte[] salt, ICryptKey key) { @@ -31,32 +32,43 @@ internal sealed class RarCryptoWrapper : Stream public int ReadAndDecrypt(byte[] buffer, int offset, int count) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) + var destination = buffer.AsSpan(offset, count); + if (count == 1 && _available > 0) { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - Span cipherText = stackalloc byte[16]; - for (var i = 0; i < alignedSize / 16; i++) + destination[0] = _buffer[_offset++]; + _available--; + return 1; + } + return ReadAndDecrypt(destination); + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override int Read(Span buffer) => ReadAndDecrypt(buffer); +#endif + + private int ReadAndDecrypt(Span buffer) + { + var count = buffer.Length; + while (!buffer.IsEmpty) + { + if (_available == 0) { - if (!_actualStream.ReadFully(cipherText)) + // Round only the requested bytes to an AES block; never read ahead + // into the next entry. The buffer bounds memory independently of read size. + var alignedSize = PrepareBuffer(buffer.Length); + if (!_actualStream.ReadFully(_buffer.AsSpan(0, alignedSize))) { - // Preserve the synchronous API's malformed-archive exception type. throw new InvalidFormatException("Unexpected end of encrypted stream"); } - - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } + _rijndael.ProcessBlocks(_buffer, alignedSize); + _offset = 0; + _available = alignedSize; } - } - - for (var i = 0; i < count; i++) - { - buffer[offset + i] = _data.Dequeue(); + var length = Math.Min(buffer.Length, _available); + _buffer.AsSpan(_offset, length).CopyTo(buffer); + buffer = buffer.Slice(length); + _offset += length; + _available -= length; } return count; } @@ -66,75 +78,94 @@ internal sealed class RarCryptoWrapper : Stream int offset, int count, CancellationToken cancellationToken - ) => ReadAndDecryptAsync(buffer, offset, count, cancellationToken).AsTask(); + ) + { + cancellationToken.ThrowIfCancellationRequested(); + var destination = buffer.AsMemory(offset, count); + // Most small reads can complete without entering the async state machine. + if (count <= _available) + { + CopyBuffered(destination.Span); + return Task.FromResult(count); + } + return ReadAndDecryptAsync(destination, cancellationToken).AsTask(); + } + + private void CopyBuffered(Span destination) + { + if (destination.Length == 1) + { + destination[0] = _buffer[_offset]; + } + else + { + _buffer.AsSpan(_offset, destination.Length).CopyTo(destination); + } + _offset += destination.Length; + _available -= destination.Length; + } + + private int PrepareBuffer(int count) + { + var alignedSize = (Math.Min(count, 8192) + 15) & ~15; + if (_buffer.Length < alignedSize) + { + // Keep tiny entries cheap, growing only when callers request bulk reads. + Array.Clear(_buffer, 0, _buffer.Length); + _buffer = new byte[alignedSize]; + } + return alignedSize; + } private async ValueTask ReadAndDecryptAsync( - byte[] buffer, - int offset, - int count, + Memory buffer, CancellationToken cancellationToken ) { cancellationToken.ThrowIfCancellationRequested(); - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) + var count = buffer.Length; + while (!buffer.IsEmpty) { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - byte[] cipherText = new byte[16]; - - try + if (_available == 0) { - for (var i = 0; i < alignedSize / 16; i++) + var alignedSize = PrepareBuffer(buffer.Length); + try { await _actualStream - .ReadExactAsync(cipherText, 0, 16, cancellationToken) + .ReadExactAsync(_buffer, 0, alignedSize, cancellationToken) .ConfigureAwait(false); - - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } } + catch (IncompleteArchiveException e) + { + throw new InvalidFormatException("Unexpected end of encrypted stream", e); + } + _rijndael.ProcessBlocks(_buffer, alignedSize); + _offset = 0; + _available = alignedSize; } - catch (IncompleteArchiveException e) - { - throw new InvalidFormatException("Unexpected end of encrypted stream", e); - } + var length = Math.Min(buffer.Length, _available); + _buffer.AsMemory(_offset, length).CopyTo(buffer); + buffer = buffer.Slice(length); + _offset += length; + _available -= length; } - - var bytesToReturn = Math.Min(count, _data.Count); - for (var i = 0; i < bytesToReturn; i++) - { - buffer[offset + i] = _data.Dequeue(); - } - - return bytesToReturn; + return count; } #if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER - public override async ValueTask ReadAsync( + public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default ) { cancellationToken.ThrowIfCancellationRequested(); - - var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); - try + if (buffer.Length <= _available) { - var bytesRead = await ReadAndDecryptAsync(array, 0, buffer.Length, cancellationToken) - .ConfigureAwait(false); - new ReadOnlySpan(array, 0, bytesRead).CopyTo(buffer.Span); - return bytesRead; - } - finally - { - System.Buffers.ArrayPool.Shared.Return(array); + CopyBuffered(buffer.Span); + return new ValueTask(buffer.Length); } + return ReadAndDecryptAsync(buffer, cancellationToken); } #endif @@ -156,6 +187,7 @@ internal sealed class RarCryptoWrapper : Stream if (disposing) { _rijndael.Dispose(); + Array.Clear(_buffer, 0, _buffer.Length); } base.Dispose(disposing); diff --git a/src/SharpCompress/Crypto/BlockTransformer.cs b/src/SharpCompress/Crypto/BlockTransformer.cs index d3196aaa..f41a8d2e 100644 --- a/src/SharpCompress/Crypto/BlockTransformer.cs +++ b/src/SharpCompress/Crypto/BlockTransformer.cs @@ -5,13 +5,10 @@ namespace SharpCompress.Crypto; internal class BlockTransformer(ICryptoTransform transformer) : IDisposable { - public byte[] ProcessBlock(ReadOnlySpan cipherText) - { - var decryptedBytes = new byte[cipherText.Length]; - transformer.TransformBlock(cipherText.ToArray(), 0, cipherText.Length, decryptedBytes, 0); + // AES supports in-place transformation. Callers retain their own bounded buffers + // instead of allocating input and output arrays for every 16-byte block. + public void ProcessBlocks(byte[] buffer, int count) => + transformer.TransformBlock(buffer, 0, count, buffer, 0); - return decryptedBytes; - } - - public void Dispose() { } + public void Dispose() => transformer.Dispose(); } diff --git a/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs b/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs new file mode 100644 index 00000000..ccfcc478 --- /dev/null +++ b/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs @@ -0,0 +1,104 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using SharpCompress.Archives.Rar; +using SharpCompress.Common.Rar.Headers; +using SharpCompress.IO; +using SharpCompress.Readers; + +namespace SharpCompress.Performance.Benchmarks; + +[MemoryDiagnoser] +public class RarEncryptedBenchmarks : ArchiveBenchmarkBase +{ + private byte[] archiveBytes = null!; + private RarHeaderFactory factory = null!; + + [Params("Rar5.encrypted_filesAndHeader.rar", "Rar5.encrypted_filesOnly.rar", "Rar5.rar")] + public string ArchiveName { get; set; } = null!; + + [GlobalSetup] + public void Setup() + { + archiveBytes = File.ReadAllBytes(GetArchivePath(ArchiveName)); + factory = CreateFactory(); + // Separate warm-cache parsing from expensive first-use key derivation. + HeadersWarm(); + } + + private static RarHeaderFactory CreateFactory() => + new(StreamingMode.Seekable, ReaderOptions.ForExternalStream.WithPassword("test")); + + [Benchmark] + public int HeadersWarm() => ReadHeaders(factory); + + [Benchmark] + public int HeadersCold() => ReadHeaders(CreateFactory()); + + private int ReadHeaders(RarHeaderFactory headerFactory) + { + using var stream = new MemoryStream(archiveBytes); + var count = 0; + foreach (var header in headerFactory.ReadHeaders(stream)) + { + count++; + } + return count; + } + + [Benchmark] + public async Task HeadersWarmAsync() + { + using var stream = new MemoryStream(archiveBytes); + var count = 0; + await foreach (var header in factory.ReadHeadersAsync(stream)) + { + count++; + } + return count; + } + + [Benchmark] + public void Extract() + { + using var stream = new MemoryStream(archiveBytes); + using var archive = RarArchive.OpenArchive( + stream, + ReaderOptions.ForExternalStream.WithPassword("test") + ); + foreach (var entry in archive.Entries) + { + if (!entry.IsDirectory) + { + using var entryStream = entry.OpenEntryStream(); + entryStream.CopyTo(Stream.Null); + } + } + } + + [Benchmark] + public async Task ExtractAsync() + { + using var stream = new MemoryStream(archiveBytes); + await using var archive = await RarArchive + .OpenAsyncArchive( + stream, + ReaderOptions.ForExternalStream.WithPassword("test"), + CancellationToken.None + ) + .ConfigureAwait(false); + await foreach (var entry in archive.EntriesAsync) + { + if (!entry.IsDirectory) + { + await using var entryStream = await entry + .OpenEntryStreamAsync(CancellationToken.None) + .ConfigureAwait(false); + await entryStream + .CopyToAsync(Stream.Null, CancellationToken.None) + .ConfigureAwait(false); + } + } + } +} diff --git a/tests/SharpCompress.Performance/rar-crypto-profiling.md b/tests/SharpCompress.Performance/rar-crypto-profiling.md new file mode 100644 index 00000000..ce20b5ef --- /dev/null +++ b/tests/SharpCompress.Performance/rar-crypto-profiling.md @@ -0,0 +1,65 @@ +# Encrypted RAR allocation profiling + +Baseline: `e61a335d` (the RAR5 HMAC allocation fix), compared with the buffered +decryption follow-up. Measurements were taken on macOS 27.0.1 ARM64 / .NET 10.0.13, +Release mode, with tiered compilation disabled. These are not Alpine results. + +## Allocation-focused measurements + +An isolated harness exercised `RarCryptoWrapper` over 1 MiB of AES-CBC ciphertext +with a fixed key (excluding PBKDF2), plus repeated `RarHeaderFactory` scans of +existing in-memory fixtures. Each workload was warmed up, then measured five +times. Tables show median elapsed time and median managed bytes allocated using +process-wide counters, including wrapper/reader construction and disposal but +excluding fixture/encryption/output-buffer setup. No other workloads ran during +the final measurements. Header scans reused a factory to exercise warm KDF caches. + +| Workload | Before bytes | After bytes | Before ms | After ms | +| --- | ---: | ---: | ---: | ---: | +| Decrypt 1 MiB, sync, 1-byte reads | 5,244,392 | 1,416 | 7.972 | 4.010 | +| Decrypt 1 MiB, async, 1-byte reads | 7,865,832 | 1,416 | 15.281 | 8.147 | +| Decrypt 1 MiB, sync, 16-byte reads | 5,244,392 | 1,416 | 6.330 | 2.899 | +| Decrypt 1 MiB, async, 16-byte reads | 12,584,424 | 4,720,008 | 8.346 | 5.929 | +| Decrypt 1 MiB, sync, 8192-byte reads | 5,260,960 | 9,632 | 6.705 | 0.110 | +| Decrypt 1 MiB, async, 8192-byte reads | 5,275,296 | 18,512 | 8.132 | 0.138 | +| RAR5 encrypted headers, sync, 100 scans | 2,425,456 | 1,403,056 | 1.331 | 1.169 | +| RAR5 encrypted headers, async, 100 scans | 2,552,656 | 1,530,256 | 3.250 | 2.983 | +| RAR4 encrypted headers, sync, 1 scan | 34,908,464 | 34,903,512 | 98.269 | 105.439 | +| RAR4 encrypted headers, async, 1 scan | 34,909,488 | 34,904,768 | 97.998 | 106.596 | +| Unencrypted RAR5 headers, sync, 100 scans | 518,608 | 518,608 | 0.217 | 0.236 | +| Unencrypted RAR5 headers, async, 100 scans | 597,808 | 597,808 | 2.289 | 2.340 | + +Async decryption here uses the `Task` array overload. Returning a non-cached +integer such as 16 still allocates a Task per call; the `Memory` overload +uses `ValueTask` and avoids that conversion. This distinction matters when +interpreting the remaining allocations. + +## Changes and remaining opportunities + +- Transform AES blocks in place rather than allocating two arrays per block. +- Batch file-data transformations in a reusable buffer bounded to 8 KiB; start + with 16 bytes to avoid imposing the larger allocation on tiny entries. +- Remove plaintext queues, per-byte header-result arrays, per-block ciphertext + arrays, and the pooled-copy adapter for `Memory` reads. +- Dispose owned crypto transforms and per-header readers deterministically. + Native memory retention was not quantified by these managed allocation counters. +- Leave the RAR3/4 KDF unchanged: its large working array dominates the roughly + 35 MB per encrypted-header scan. The measured legacy workloads were slightly + slower, not faster. Any KDF rewrite needs separate correctness/performance work. +- Primitive-read arrays and fixed KDF/cache setup costs remain secondary targets. + +## Repeatable repository benchmarks + +`RarEncryptedBenchmarks` provides warm/cold header parsing, async warm parsing, +and sync/async archive extraction, with encrypted-header, encrypted-data-only, +and unencrypted fixtures. Load fixture bytes outside measured iterations. + +```sh +dotnet run --project tests/SharpCompress.Performance -c Release -- --filter '*RarEncryptedBenchmarks*' +``` + +All 15 benchmark cases were smoke-tested with one warmup and one measured +invocation; those smoke-test timings are not statistically meaningful and are +not the before/after measurements above. Dedicated regression tests cover +bounded allocations, CRC/read counts, block boundaries, large and fragmented +reads, span/memory overloads, stream ownership, and transform disposal. diff --git a/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs b/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs index edb814bd..fea920ad 100644 --- a/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs +++ b/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar; +using SharpCompress.Crypto; using SharpCompress.IO; using SharpCompress.Test.Mocks; using Xunit; @@ -14,6 +15,165 @@ namespace SharpCompress.Test.Rar; public class RarCryptoWrapperTests { + [Theory] + [InlineData(false, 1)] + [InlineData(true, 1)] + [InlineData(false, 15)] + [InlineData(true, 17)] + [InlineData(false, 8191)] + [InlineData(true, 8192)] + [InlineData(false, 8193)] + [InlineData(true, 32784)] + public async Task Rar5_Crypto_Reads_Large_Data_Without_Reading_Next_Entry( + bool useAsync, + int readSize + ) + { + var plaintext = Enumerable.Range(0, 32784).Select(i => (byte)i).ToArray(); + var info = CreateCryptoInfo(); + var ciphertext = Encrypt(info, plaintext); + using var input = new MemoryStream(ciphertext.Concat(new byte[16]).ToArray()); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var result = new byte[plaintext.Length + 2]; + var offset = 0; + while (offset < plaintext.Length) + { + var count = Math.Min(readSize, plaintext.Length - offset); + var read = useAsync + ? await crypto.ReadAsync(result, offset + 1, count, CancellationToken.None) + : crypto.Read(result, offset + 1, count); + Assert.Equal(count, read); + offset += read; + Assert.Equal((offset + 15) & ~15, input.Position); + } + Assert.Equal(plaintext, result.Skip(1).Take(plaintext.Length)); + Assert.Equal(0, result[0]); + Assert.Equal(0, result[result.Length - 1]); + Assert.Equal(ciphertext.Length, input.Position); + } + +#if !LEGACY_DOTNET + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Rar5_Crypto_Reads_Span_And_Memory(bool useAsync) + { + var info = CreateCryptoInfo(); + using var input = new MemoryStream(Encrypt(info)); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var output = new byte[32]; + for (var i = 0; i < output.Length; i++) + { + var read = useAsync + ? await crypto.ReadAsync(output.AsMemory(i, 1), CancellationToken.None) + : crypto.Read(output.AsSpan(i, 1)); + Assert.Equal(1, read); + } + Assert.Equal(Plaintext, output); + } + + [Fact] + public void Rar5_Crypto_DoesNotAllocate_PerBlock() + { + var info = CreateCryptoInfo(); + var plaintext = new byte[1024 * 1024]; + using var input = new MemoryStream(Encrypt(info, plaintext)); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var buffer = new byte[8192]; + Assert.Equal(buffer.Length, crypto.Read(buffer, 0, buffer.Length)); + var before = GC.GetAllocatedBytesForCurrentThread(); + var total = 0; + for (var i = buffer.Length; i < plaintext.Length; i += buffer.Length) + { + total += crypto.Read(buffer, 0, buffer.Length); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.Equal(plaintext.Length - buffer.Length, total); + Assert.True(allocated < 16 * 1024, $"Allocated {allocated} bytes."); + } +#endif + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Rar5_Crypto_Header_Reads_Track_Crc_And_Leave_Stream_Open(bool useAsync) + { + var info = CreateCryptoInfo(); + using var input = new ShortReadStream(new MemoryStream(Encrypt(info))); + uint expectedCrc = SharpCompress.Compressors.Rar.RarCRC.CheckCrc( + 0xffffffff, + Plaintext, + 0, + Plaintext.Length + ); + if (useAsync) + { + using var asyncOnly = new AsyncOnlyStream(input, disposeStream: false); + using var reader = await AsyncRarCryptoBinaryReader.Create( + asyncOnly, + new CryptKey5("test", info), + info.Salt + ); + reader.ResetCrc(); + Assert.Equal(Plaintext[0], await reader.ReadByteAsync()); + Assert.Equal(Plaintext.Skip(1), await reader.ReadBytesAsync(31)); + Assert.Equal(32, reader.CurrentReadByteCount); + Assert.Equal(~expectedCrc, reader.GetCrc32()); + } + else + { + using var reader = RarCryptoBinaryReader.Create( + input, + new CryptKey5("test", info), + info.Salt + ); + reader.ResetCrc(); + Assert.Equal(Plaintext[0], reader.ReadByte()); + Assert.Equal(Plaintext.Skip(1), reader.ReadBytes(31)); + Assert.Equal(32, reader.CurrentReadByteCount); + Assert.Equal(~expectedCrc, reader.GetCrc32()); + } + Assert.False(input.IsDisposed); + } + + [Fact] + public void Rar_Crypto_BlockTransformer_Disposes_Transform() + { + var transform = new TrackingTransform(); + using (var block = new BlockTransformer(transform)) + { + block.ProcessBlocks(new byte[32], 32); + Assert.False(transform.IsDisposed); + } + Assert.True(transform.IsDisposed); + } + + private sealed class TrackingTransform : ICryptoTransform + { + public bool IsDisposed { get; private set; } + public int InputBlockSize => 16; + public int OutputBlockSize => 16; + public bool CanTransformMultipleBlocks => true; + public bool CanReuseTransform => false; + + public int TransformBlock( + byte[] inputBuffer, + int inputOffset, + int inputCount, + byte[] outputBuffer, + int outputOffset + ) + { + Buffer.BlockCopy(inputBuffer, inputOffset, outputBuffer, outputOffset, inputCount); + return inputCount; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) => + throw new NotSupportedException(); + + public void Dispose() => IsDisposed = true; + } + private static readonly byte[] Plaintext = Enumerable .Range(0, 32) .Select(x => (byte)x) @@ -108,13 +268,14 @@ public class RarCryptoWrapperTests return Rar5CryptoInfo.Create(reader, true); } - private static byte[] Encrypt(Rar5CryptoInfo info) + private static byte[] Encrypt(Rar5CryptoInfo info, byte[]? plaintext = null) { using var aes = Aes.Create(); aes.Key = CryptKey5.GenerateRarPBKDF2Key("test", info.Salt, 1)[0]; aes.IV = info.InitV; aes.Padding = PaddingMode.None; using var encryptor = aes.CreateEncryptor(); - return encryptor.TransformFinalBlock(Plaintext, 0, Plaintext.Length); + plaintext ??= Plaintext; + return encryptor.TransformFinalBlock(plaintext, 0, plaintext.Length); } }