From f63cd81a65441ff13da586758ff337d77e227933 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 20 Oct 2026 10:43:28 +0100 Subject: [PATCH] Reduce allocations in encrypted RAR reading Reuse bounded buffers and transform AES blocks in place. Remove per-block arrays and plaintext queues, avoid per-byte header allocations, and dispose owned crypto transforms. Add encrypted RAR benchmarks, regression tests, and before/after profiling results. Full .NET 10 suite passes (2,446 tests); all library targets build and net48 tests compile. --- .../Common/Rar/AsyncRarCryptoBinaryReader.cs | 68 ++++--- .../Rar/Headers/RarHeaderFactory.Async.cs | 2 + .../Common/Rar/Headers/RarHeaderFactory.cs | 2 + .../Common/Rar/RarCryptoBinaryReader.cs | 58 +++--- .../Common/Rar/RarCryptoWrapper.cs | 166 +++++++++++------- src/SharpCompress/Crypto/BlockTransformer.cs | 13 +- .../Benchmarks/RarEncryptedBenchmarks.cs | 104 +++++++++++ .../rar-crypto-profiling.md | 65 +++++++ .../Rar/RarCryptoWrapperTests.cs | 165 ++++++++++++++++- 9 files changed, 511 insertions(+), 132 deletions(-) create mode 100644 tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs create mode 100644 tests/SharpCompress.Performance/rar-crypto-profiling.md diff --git a/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs b/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs index c3f5d005..c79896a3 100644 --- a/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs +++ b/src/SharpCompress/Common/Rar/AsyncRarCryptoBinaryReader.cs @@ -1,17 +1,18 @@ using System; -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; using SharpCompress.Crypto; +using SharpCompress.IO; namespace SharpCompress.Common.Rar; internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader { private BlockTransformer _rijndael = default!; - private readonly Queue _data = new(); + private readonly byte[] _block = new byte[16]; + private int _blockOffset = 16; private long _readCount; private AsyncRarCryptoBinaryReader(Stream stream) @@ -50,8 +51,26 @@ internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader CancellationToken cancellationToken = default ) { - var bytes = await ReadAndDecryptBytesAsync(1, cancellationToken).ConfigureAwait(false); - return bytes[0]; + cancellationToken.ThrowIfCancellationRequested(); + if (_blockOffset == _block.Length) + { + try + { + await BaseStream + .ReadExactAsync(_block, 0, _block.Length, cancellationToken) + .ConfigureAwait(false); + } + catch (IncompleteArchiveException e) + { + throw new InvalidFormatException("Unexpected end of encrypted stream", e); + } + _rijndael.ProcessBlocks(_block, _block.Length); + _blockOffset = 0; + } + var value = _block[_blockOffset++]; + UpdateCrc(value); + _readCount++; + return value; } private ValueTask ReadBytesAsyncBase(int count) => base.ReadBytesAsync(count); @@ -69,43 +88,36 @@ internal sealed class AsyncRarCryptoBinaryReader : AsyncRarCrcBinaryReader CancellationToken cancellationToken ) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) - { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - for (var i = 0; i < alignedSize / 16; i++) - { - var cipherText = await ReadBytesNoCrcAsync(16, cancellationToken) - .ConfigureAwait(false); - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } - } - } - var decryptedBytes = new byte[count]; for (var i = 0; i < count; i++) { - var b = _data.Dequeue(); - decryptedBytes[i] = b; - UpdateCrc(b); + decryptedBytes[i] = await ReadByteAsync(cancellationToken).ConfigureAwait(false); } - _readCount += count; return decryptedBytes; } - public void ClearQueue() => _data.Clear(); + public void ClearQueue() => _blockOffset = _block.Length; public void SkipQueue() { var position = BaseStream.Position; - BaseStream.Position = position + _data.Count; + BaseStream.Position = position + _block.Length - _blockOffset; ClearQueue(); } + + public override void Dispose() + { + _rijndael.Dispose(); + base.Dispose(); + } + +#if NET8_0_OR_GREATER + public override ValueTask DisposeAsync() + { + _rijndael.Dispose(); + return base.DisposeAsync(); + } +#endif } diff --git a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs index a6108b32..85f69db4 100644 --- a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.Async.cs @@ -79,6 +79,8 @@ public partial class RarHeaderFactory } } + // Readers leave the underlying stream open but own their crypto transforms. + using var readerLifetime = reader; var header = await RarHeader .TryReadBaseAsync(reader, _isRar5, Options.ArchiveEncoding, cancellationToken) .ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs index 27ae9db6..4fb9c00b 100644 --- a/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs +++ b/src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs @@ -98,6 +98,8 @@ public partial class RarHeaderFactory } } + // Readers leave the underlying stream open but own their crypto transforms. + using var readerLifetime = reader; var header = RarHeader.TryReadBase(reader, _isRar5, Options.ArchiveEncoding); if (header is null) { diff --git a/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs b/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs index 319cf4c5..60299057 100644 --- a/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs +++ b/src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs @@ -1,14 +1,15 @@ -using System.Collections.Generic; using System.IO; using SharpCompress.Common.Rar.Headers; using SharpCompress.Crypto; +using SharpCompress.IO; namespace SharpCompress.Common.Rar; internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader { private BlockTransformer _rijndael = default!; - private readonly Queue _data = new(); + private readonly byte[] _block = new byte[16]; + private int _blockOffset = 16; private long _readCount; private RarCryptoBinaryReader(Stream stream) @@ -42,7 +43,22 @@ internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader public override void Mark() => _readCount = 0; - public override byte ReadByte() => ReadAndDecryptBytes(1)[0]; + public override byte ReadByte() + { + if (_blockOffset == _block.Length) + { + if (!BaseStream.ReadFully(_block)) + { + throw new InvalidFormatException("Unexpected end of encrypted stream"); + } + _rijndael.ProcessBlocks(_block, _block.Length); + _blockOffset = 0; + } + var value = _block[_blockOffset++]; + UpdateCrc(value); + _readCount++; + return value; + } public override byte[] ReadBytes(int count) => ReadAndDecryptBytes(count); @@ -50,43 +66,31 @@ internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader private byte[] ReadAndDecryptBytes(int count) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) - { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - for (var i = 0; i < alignedSize / 16; i++) - { - //long ax = System.currentTimeMillis(); - var cipherText = ReadBytesNoCrc(16); - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } - } - } - var decryptedBytes = new byte[count]; for (var i = 0; i < count; i++) { - var b = _data.Dequeue(); - decryptedBytes[i] = b; - UpdateCrc(b); + decryptedBytes[i] = ReadByte(); } - _readCount += count; return decryptedBytes; } - public void ClearQueue() => _data.Clear(); + public void ClearQueue() => _blockOffset = _block.Length; public void SkipQueue() { var position = BaseStream.Position; - BaseStream.Position = position + _data.Count; + BaseStream.Position = position + _block.Length - _blockOffset; ClearQueue(); } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _rijndael.Dispose(); + } + base.Dispose(disposing); + } } diff --git a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs index d5e7112b..55bc7425 100644 --- a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs +++ b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -11,8 +10,10 @@ namespace SharpCompress.Common.Rar; internal sealed class RarCryptoWrapper : Stream { private readonly Stream _actualStream; - private BlockTransformer _rijndael; - private readonly Queue _data = new(); + private readonly BlockTransformer _rijndael; + private byte[] _buffer = new byte[16]; + private int _offset; + private int _available; public RarCryptoWrapper(Stream actualStream, byte[] salt, ICryptKey key) { @@ -31,32 +32,43 @@ internal sealed class RarCryptoWrapper : Stream public int ReadAndDecrypt(byte[] buffer, int offset, int count) { - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) + var destination = buffer.AsSpan(offset, count); + if (count == 1 && _available > 0) { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - Span cipherText = stackalloc byte[16]; - for (var i = 0; i < alignedSize / 16; i++) + destination[0] = _buffer[_offset++]; + _available--; + return 1; + } + return ReadAndDecrypt(destination); + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override int Read(Span buffer) => ReadAndDecrypt(buffer); +#endif + + private int ReadAndDecrypt(Span buffer) + { + var count = buffer.Length; + while (!buffer.IsEmpty) + { + if (_available == 0) { - if (!_actualStream.ReadFully(cipherText)) + // Round only the requested bytes to an AES block; never read ahead + // into the next entry. The buffer bounds memory independently of read size. + var alignedSize = PrepareBuffer(buffer.Length); + if (!_actualStream.ReadFully(_buffer.AsSpan(0, alignedSize))) { - // Preserve the synchronous API's malformed-archive exception type. throw new InvalidFormatException("Unexpected end of encrypted stream"); } - - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } + _rijndael.ProcessBlocks(_buffer, alignedSize); + _offset = 0; + _available = alignedSize; } - } - - for (var i = 0; i < count; i++) - { - buffer[offset + i] = _data.Dequeue(); + var length = Math.Min(buffer.Length, _available); + _buffer.AsSpan(_offset, length).CopyTo(buffer); + buffer = buffer.Slice(length); + _offset += length; + _available -= length; } return count; } @@ -66,75 +78,94 @@ internal sealed class RarCryptoWrapper : Stream int offset, int count, CancellationToken cancellationToken - ) => ReadAndDecryptAsync(buffer, offset, count, cancellationToken).AsTask(); + ) + { + cancellationToken.ThrowIfCancellationRequested(); + var destination = buffer.AsMemory(offset, count); + // Most small reads can complete without entering the async state machine. + if (count <= _available) + { + CopyBuffered(destination.Span); + return Task.FromResult(count); + } + return ReadAndDecryptAsync(destination, cancellationToken).AsTask(); + } + + private void CopyBuffered(Span destination) + { + if (destination.Length == 1) + { + destination[0] = _buffer[_offset]; + } + else + { + _buffer.AsSpan(_offset, destination.Length).CopyTo(destination); + } + _offset += destination.Length; + _available -= destination.Length; + } + + private int PrepareBuffer(int count) + { + var alignedSize = (Math.Min(count, 8192) + 15) & ~15; + if (_buffer.Length < alignedSize) + { + // Keep tiny entries cheap, growing only when callers request bulk reads. + Array.Clear(_buffer, 0, _buffer.Length); + _buffer = new byte[alignedSize]; + } + return alignedSize; + } private async ValueTask ReadAndDecryptAsync( - byte[] buffer, - int offset, - int count, + Memory buffer, CancellationToken cancellationToken ) { cancellationToken.ThrowIfCancellationRequested(); - var queueSize = _data.Count; - var sizeToRead = count - queueSize; - - if (sizeToRead > 0) + var count = buffer.Length; + while (!buffer.IsEmpty) { - var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf); - byte[] cipherText = new byte[16]; - - try + if (_available == 0) { - for (var i = 0; i < alignedSize / 16; i++) + var alignedSize = PrepareBuffer(buffer.Length); + try { await _actualStream - .ReadExactAsync(cipherText, 0, 16, cancellationToken) + .ReadExactAsync(_buffer, 0, alignedSize, cancellationToken) .ConfigureAwait(false); - - var readBytes = _rijndael.ProcessBlock(cipherText); - foreach (var readByte in readBytes) - { - _data.Enqueue(readByte); - } } + catch (IncompleteArchiveException e) + { + throw new InvalidFormatException("Unexpected end of encrypted stream", e); + } + _rijndael.ProcessBlocks(_buffer, alignedSize); + _offset = 0; + _available = alignedSize; } - catch (IncompleteArchiveException e) - { - throw new InvalidFormatException("Unexpected end of encrypted stream", e); - } + var length = Math.Min(buffer.Length, _available); + _buffer.AsMemory(_offset, length).CopyTo(buffer); + buffer = buffer.Slice(length); + _offset += length; + _available -= length; } - - var bytesToReturn = Math.Min(count, _data.Count); - for (var i = 0; i < bytesToReturn; i++) - { - buffer[offset + i] = _data.Dequeue(); - } - - return bytesToReturn; + return count; } #if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER - public override async ValueTask ReadAsync( + public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default ) { cancellationToken.ThrowIfCancellationRequested(); - - var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); - try + if (buffer.Length <= _available) { - var bytesRead = await ReadAndDecryptAsync(array, 0, buffer.Length, cancellationToken) - .ConfigureAwait(false); - new ReadOnlySpan(array, 0, bytesRead).CopyTo(buffer.Span); - return bytesRead; - } - finally - { - System.Buffers.ArrayPool.Shared.Return(array); + CopyBuffered(buffer.Span); + return new ValueTask(buffer.Length); } + return ReadAndDecryptAsync(buffer, cancellationToken); } #endif @@ -156,6 +187,7 @@ internal sealed class RarCryptoWrapper : Stream if (disposing) { _rijndael.Dispose(); + Array.Clear(_buffer, 0, _buffer.Length); } base.Dispose(disposing); diff --git a/src/SharpCompress/Crypto/BlockTransformer.cs b/src/SharpCompress/Crypto/BlockTransformer.cs index d3196aaa..f41a8d2e 100644 --- a/src/SharpCompress/Crypto/BlockTransformer.cs +++ b/src/SharpCompress/Crypto/BlockTransformer.cs @@ -5,13 +5,10 @@ namespace SharpCompress.Crypto; internal class BlockTransformer(ICryptoTransform transformer) : IDisposable { - public byte[] ProcessBlock(ReadOnlySpan cipherText) - { - var decryptedBytes = new byte[cipherText.Length]; - transformer.TransformBlock(cipherText.ToArray(), 0, cipherText.Length, decryptedBytes, 0); + // AES supports in-place transformation. Callers retain their own bounded buffers + // instead of allocating input and output arrays for every 16-byte block. + public void ProcessBlocks(byte[] buffer, int count) => + transformer.TransformBlock(buffer, 0, count, buffer, 0); - return decryptedBytes; - } - - public void Dispose() { } + public void Dispose() => transformer.Dispose(); } diff --git a/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs b/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs new file mode 100644 index 00000000..ccfcc478 --- /dev/null +++ b/tests/SharpCompress.Performance/Benchmarks/RarEncryptedBenchmarks.cs @@ -0,0 +1,104 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using SharpCompress.Archives.Rar; +using SharpCompress.Common.Rar.Headers; +using SharpCompress.IO; +using SharpCompress.Readers; + +namespace SharpCompress.Performance.Benchmarks; + +[MemoryDiagnoser] +public class RarEncryptedBenchmarks : ArchiveBenchmarkBase +{ + private byte[] archiveBytes = null!; + private RarHeaderFactory factory = null!; + + [Params("Rar5.encrypted_filesAndHeader.rar", "Rar5.encrypted_filesOnly.rar", "Rar5.rar")] + public string ArchiveName { get; set; } = null!; + + [GlobalSetup] + public void Setup() + { + archiveBytes = File.ReadAllBytes(GetArchivePath(ArchiveName)); + factory = CreateFactory(); + // Separate warm-cache parsing from expensive first-use key derivation. + HeadersWarm(); + } + + private static RarHeaderFactory CreateFactory() => + new(StreamingMode.Seekable, ReaderOptions.ForExternalStream.WithPassword("test")); + + [Benchmark] + public int HeadersWarm() => ReadHeaders(factory); + + [Benchmark] + public int HeadersCold() => ReadHeaders(CreateFactory()); + + private int ReadHeaders(RarHeaderFactory headerFactory) + { + using var stream = new MemoryStream(archiveBytes); + var count = 0; + foreach (var header in headerFactory.ReadHeaders(stream)) + { + count++; + } + return count; + } + + [Benchmark] + public async Task HeadersWarmAsync() + { + using var stream = new MemoryStream(archiveBytes); + var count = 0; + await foreach (var header in factory.ReadHeadersAsync(stream)) + { + count++; + } + return count; + } + + [Benchmark] + public void Extract() + { + using var stream = new MemoryStream(archiveBytes); + using var archive = RarArchive.OpenArchive( + stream, + ReaderOptions.ForExternalStream.WithPassword("test") + ); + foreach (var entry in archive.Entries) + { + if (!entry.IsDirectory) + { + using var entryStream = entry.OpenEntryStream(); + entryStream.CopyTo(Stream.Null); + } + } + } + + [Benchmark] + public async Task ExtractAsync() + { + using var stream = new MemoryStream(archiveBytes); + await using var archive = await RarArchive + .OpenAsyncArchive( + stream, + ReaderOptions.ForExternalStream.WithPassword("test"), + CancellationToken.None + ) + .ConfigureAwait(false); + await foreach (var entry in archive.EntriesAsync) + { + if (!entry.IsDirectory) + { + await using var entryStream = await entry + .OpenEntryStreamAsync(CancellationToken.None) + .ConfigureAwait(false); + await entryStream + .CopyToAsync(Stream.Null, CancellationToken.None) + .ConfigureAwait(false); + } + } + } +} diff --git a/tests/SharpCompress.Performance/rar-crypto-profiling.md b/tests/SharpCompress.Performance/rar-crypto-profiling.md new file mode 100644 index 00000000..ce20b5ef --- /dev/null +++ b/tests/SharpCompress.Performance/rar-crypto-profiling.md @@ -0,0 +1,65 @@ +# Encrypted RAR allocation profiling + +Baseline: `e61a335d` (the RAR5 HMAC allocation fix), compared with the buffered +decryption follow-up. Measurements were taken on macOS 27.0.1 ARM64 / .NET 10.0.13, +Release mode, with tiered compilation disabled. These are not Alpine results. + +## Allocation-focused measurements + +An isolated harness exercised `RarCryptoWrapper` over 1 MiB of AES-CBC ciphertext +with a fixed key (excluding PBKDF2), plus repeated `RarHeaderFactory` scans of +existing in-memory fixtures. Each workload was warmed up, then measured five +times. Tables show median elapsed time and median managed bytes allocated using +process-wide counters, including wrapper/reader construction and disposal but +excluding fixture/encryption/output-buffer setup. No other workloads ran during +the final measurements. Header scans reused a factory to exercise warm KDF caches. + +| Workload | Before bytes | After bytes | Before ms | After ms | +| --- | ---: | ---: | ---: | ---: | +| Decrypt 1 MiB, sync, 1-byte reads | 5,244,392 | 1,416 | 7.972 | 4.010 | +| Decrypt 1 MiB, async, 1-byte reads | 7,865,832 | 1,416 | 15.281 | 8.147 | +| Decrypt 1 MiB, sync, 16-byte reads | 5,244,392 | 1,416 | 6.330 | 2.899 | +| Decrypt 1 MiB, async, 16-byte reads | 12,584,424 | 4,720,008 | 8.346 | 5.929 | +| Decrypt 1 MiB, sync, 8192-byte reads | 5,260,960 | 9,632 | 6.705 | 0.110 | +| Decrypt 1 MiB, async, 8192-byte reads | 5,275,296 | 18,512 | 8.132 | 0.138 | +| RAR5 encrypted headers, sync, 100 scans | 2,425,456 | 1,403,056 | 1.331 | 1.169 | +| RAR5 encrypted headers, async, 100 scans | 2,552,656 | 1,530,256 | 3.250 | 2.983 | +| RAR4 encrypted headers, sync, 1 scan | 34,908,464 | 34,903,512 | 98.269 | 105.439 | +| RAR4 encrypted headers, async, 1 scan | 34,909,488 | 34,904,768 | 97.998 | 106.596 | +| Unencrypted RAR5 headers, sync, 100 scans | 518,608 | 518,608 | 0.217 | 0.236 | +| Unencrypted RAR5 headers, async, 100 scans | 597,808 | 597,808 | 2.289 | 2.340 | + +Async decryption here uses the `Task` array overload. Returning a non-cached +integer such as 16 still allocates a Task per call; the `Memory` overload +uses `ValueTask` and avoids that conversion. This distinction matters when +interpreting the remaining allocations. + +## Changes and remaining opportunities + +- Transform AES blocks in place rather than allocating two arrays per block. +- Batch file-data transformations in a reusable buffer bounded to 8 KiB; start + with 16 bytes to avoid imposing the larger allocation on tiny entries. +- Remove plaintext queues, per-byte header-result arrays, per-block ciphertext + arrays, and the pooled-copy adapter for `Memory` reads. +- Dispose owned crypto transforms and per-header readers deterministically. + Native memory retention was not quantified by these managed allocation counters. +- Leave the RAR3/4 KDF unchanged: its large working array dominates the roughly + 35 MB per encrypted-header scan. The measured legacy workloads were slightly + slower, not faster. Any KDF rewrite needs separate correctness/performance work. +- Primitive-read arrays and fixed KDF/cache setup costs remain secondary targets. + +## Repeatable repository benchmarks + +`RarEncryptedBenchmarks` provides warm/cold header parsing, async warm parsing, +and sync/async archive extraction, with encrypted-header, encrypted-data-only, +and unencrypted fixtures. Load fixture bytes outside measured iterations. + +```sh +dotnet run --project tests/SharpCompress.Performance -c Release -- --filter '*RarEncryptedBenchmarks*' +``` + +All 15 benchmark cases were smoke-tested with one warmup and one measured +invocation; those smoke-test timings are not statistically meaningful and are +not the before/after measurements above. Dedicated regression tests cover +bounded allocations, CRC/read counts, block boundaries, large and fragmented +reads, span/memory overloads, stream ownership, and transform disposal. diff --git a/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs b/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs index edb814bd..fea920ad 100644 --- a/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs +++ b/tests/SharpCompress.Test/Rar/RarCryptoWrapperTests.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar; +using SharpCompress.Crypto; using SharpCompress.IO; using SharpCompress.Test.Mocks; using Xunit; @@ -14,6 +15,165 @@ namespace SharpCompress.Test.Rar; public class RarCryptoWrapperTests { + [Theory] + [InlineData(false, 1)] + [InlineData(true, 1)] + [InlineData(false, 15)] + [InlineData(true, 17)] + [InlineData(false, 8191)] + [InlineData(true, 8192)] + [InlineData(false, 8193)] + [InlineData(true, 32784)] + public async Task Rar5_Crypto_Reads_Large_Data_Without_Reading_Next_Entry( + bool useAsync, + int readSize + ) + { + var plaintext = Enumerable.Range(0, 32784).Select(i => (byte)i).ToArray(); + var info = CreateCryptoInfo(); + var ciphertext = Encrypt(info, plaintext); + using var input = new MemoryStream(ciphertext.Concat(new byte[16]).ToArray()); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var result = new byte[plaintext.Length + 2]; + var offset = 0; + while (offset < plaintext.Length) + { + var count = Math.Min(readSize, plaintext.Length - offset); + var read = useAsync + ? await crypto.ReadAsync(result, offset + 1, count, CancellationToken.None) + : crypto.Read(result, offset + 1, count); + Assert.Equal(count, read); + offset += read; + Assert.Equal((offset + 15) & ~15, input.Position); + } + Assert.Equal(plaintext, result.Skip(1).Take(plaintext.Length)); + Assert.Equal(0, result[0]); + Assert.Equal(0, result[result.Length - 1]); + Assert.Equal(ciphertext.Length, input.Position); + } + +#if !LEGACY_DOTNET + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Rar5_Crypto_Reads_Span_And_Memory(bool useAsync) + { + var info = CreateCryptoInfo(); + using var input = new MemoryStream(Encrypt(info)); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var output = new byte[32]; + for (var i = 0; i < output.Length; i++) + { + var read = useAsync + ? await crypto.ReadAsync(output.AsMemory(i, 1), CancellationToken.None) + : crypto.Read(output.AsSpan(i, 1)); + Assert.Equal(1, read); + } + Assert.Equal(Plaintext, output); + } + + [Fact] + public void Rar5_Crypto_DoesNotAllocate_PerBlock() + { + var info = CreateCryptoInfo(); + var plaintext = new byte[1024 * 1024]; + using var input = new MemoryStream(Encrypt(info, plaintext)); + using var crypto = new RarCryptoWrapper(input, info.Salt, new CryptKey5("test", info)); + var buffer = new byte[8192]; + Assert.Equal(buffer.Length, crypto.Read(buffer, 0, buffer.Length)); + var before = GC.GetAllocatedBytesForCurrentThread(); + var total = 0; + for (var i = buffer.Length; i < plaintext.Length; i += buffer.Length) + { + total += crypto.Read(buffer, 0, buffer.Length); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.Equal(plaintext.Length - buffer.Length, total); + Assert.True(allocated < 16 * 1024, $"Allocated {allocated} bytes."); + } +#endif + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Rar5_Crypto_Header_Reads_Track_Crc_And_Leave_Stream_Open(bool useAsync) + { + var info = CreateCryptoInfo(); + using var input = new ShortReadStream(new MemoryStream(Encrypt(info))); + uint expectedCrc = SharpCompress.Compressors.Rar.RarCRC.CheckCrc( + 0xffffffff, + Plaintext, + 0, + Plaintext.Length + ); + if (useAsync) + { + using var asyncOnly = new AsyncOnlyStream(input, disposeStream: false); + using var reader = await AsyncRarCryptoBinaryReader.Create( + asyncOnly, + new CryptKey5("test", info), + info.Salt + ); + reader.ResetCrc(); + Assert.Equal(Plaintext[0], await reader.ReadByteAsync()); + Assert.Equal(Plaintext.Skip(1), await reader.ReadBytesAsync(31)); + Assert.Equal(32, reader.CurrentReadByteCount); + Assert.Equal(~expectedCrc, reader.GetCrc32()); + } + else + { + using var reader = RarCryptoBinaryReader.Create( + input, + new CryptKey5("test", info), + info.Salt + ); + reader.ResetCrc(); + Assert.Equal(Plaintext[0], reader.ReadByte()); + Assert.Equal(Plaintext.Skip(1), reader.ReadBytes(31)); + Assert.Equal(32, reader.CurrentReadByteCount); + Assert.Equal(~expectedCrc, reader.GetCrc32()); + } + Assert.False(input.IsDisposed); + } + + [Fact] + public void Rar_Crypto_BlockTransformer_Disposes_Transform() + { + var transform = new TrackingTransform(); + using (var block = new BlockTransformer(transform)) + { + block.ProcessBlocks(new byte[32], 32); + Assert.False(transform.IsDisposed); + } + Assert.True(transform.IsDisposed); + } + + private sealed class TrackingTransform : ICryptoTransform + { + public bool IsDisposed { get; private set; } + public int InputBlockSize => 16; + public int OutputBlockSize => 16; + public bool CanTransformMultipleBlocks => true; + public bool CanReuseTransform => false; + + public int TransformBlock( + byte[] inputBuffer, + int inputOffset, + int inputCount, + byte[] outputBuffer, + int outputOffset + ) + { + Buffer.BlockCopy(inputBuffer, inputOffset, outputBuffer, outputOffset, inputCount); + return inputCount; + } + + public byte[] TransformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) => + throw new NotSupportedException(); + + public void Dispose() => IsDisposed = true; + } + private static readonly byte[] Plaintext = Enumerable .Range(0, 32) .Select(x => (byte)x) @@ -108,13 +268,14 @@ public class RarCryptoWrapperTests return Rar5CryptoInfo.Create(reader, true); } - private static byte[] Encrypt(Rar5CryptoInfo info) + private static byte[] Encrypt(Rar5CryptoInfo info, byte[]? plaintext = null) { using var aes = Aes.Create(); aes.Key = CryptKey5.GenerateRarPBKDF2Key("test", info.Salt, 1)[0]; aes.IV = info.InitV; aes.Padding = PaddingMode.None; using var encryptor = aes.CreateEncryptor(); - return encryptor.TransformFinalBlock(Plaintext, 0, Plaintext.Length); + plaintext ??= Plaintext; + return encryptor.TransformFinalBlock(plaintext, 0, plaintext.Length); } }