From 700ae66e0ad1cc69d3692b8b92fb070f1274ff8d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 19 Oct 2026 11:49:08 +0100 Subject: [PATCH 1/3] Fix usage of DisposeAsyncScope --- .../Streams/BufferPoolLifetimeTests.cs | 4 +--- .../Streams/Lzma2TruncationTests.cs | 14 ++++---------- .../SharpCompress.Test/Streams/RarLifetimeTests.cs | 7 ++----- .../Streams/SevenZipLifetimeTests.cs | 14 ++++---------- .../Zip/ZipEncryptedDataDescriptorTests.cs | 14 ++++---------- .../Zip/ZipWinzipAesReaderTests.cs | 14 ++++---------- 6 files changed, 19 insertions(+), 48 deletions(-) diff --git a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs index 01a4fbbb..3cb5feac 100644 --- a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs +++ b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs @@ -103,13 +103,11 @@ public class BufferPoolLifetimeTests Assert.Equal(1, pool.ReturnCount); Assert.Single(pool.Active); -#if !LEGACY_DOTNET if (asyncDispose) { - await stream.DisposeAsync(); + await using (stream.DisposeAsyncScope().ConfigureAwait(false)) { } } else -#endif { stream.Dispose(); } diff --git a/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs index 32dd795c..f20e2d54 100644 --- a/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs +++ b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs @@ -92,11 +92,8 @@ public class Lzma2TruncationTests data[i] = Convert.ToByte(TruncatedXzHex.Substring(i * 2, 2), 16); } using var input = new MemoryStream(data); -#if LEGACY_DOTNET - using var decoder = new XZStream(input); -#else - await using var decoder = new XZStream(input); -#endif + var decoder = new XZStream(input); + await using var scope = decoder.DisposeAsyncScope().ConfigureAwait(false); if (useAsync) { await Assert.ThrowsAnyAsync(() => @@ -120,11 +117,8 @@ public class Lzma2TruncationTests await using var archive = await ArchiveFactory.OpenAsyncArchive(input); await foreach (var entry in archive.EntriesAsync) { -#if LEGACY_DOTNET - using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#else - await using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#endif + var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); + await using var scope = stream.DisposeAsyncScope().ConfigureAwait(false); await Assert.ThrowsAnyAsync(() => stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None) ); diff --git a/tests/SharpCompress.Test/Streams/RarLifetimeTests.cs b/tests/SharpCompress.Test/Streams/RarLifetimeTests.cs index eb9743d2..e9cf7ec6 100644 --- a/tests/SharpCompress.Test/Streams/RarLifetimeTests.cs +++ b/tests/SharpCompress.Test/Streams/RarLifetimeTests.cs @@ -70,11 +70,8 @@ public class RarLifetimeTests : TestBase ); await foreach (var entry in archive.EntriesAsync.Where(entry => !entry.IsDirectory)) { -#if LEGACY_DOTNET - using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#else - await using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#endif + var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); + await using var scope = stream.DisposeAsyncScope().ConfigureAwait(false); await stream.CopyToAsync(Stream.Null, Constants.BufferSize, CancellationToken.None); } diff --git a/tests/SharpCompress.Test/Streams/SevenZipLifetimeTests.cs b/tests/SharpCompress.Test/Streams/SevenZipLifetimeTests.cs index 47260178..5fe20d5a 100644 --- a/tests/SharpCompress.Test/Streams/SevenZipLifetimeTests.cs +++ b/tests/SharpCompress.Test/Streams/SevenZipLifetimeTests.cs @@ -46,11 +46,8 @@ public class SevenZipLifetimeTests : TestBase { if (!reader.Entry.IsDirectory) { -#if LEGACY_DOTNET - using var stream = await reader.OpenEntryStreamAsync(CancellationToken.None); -#else - await using var stream = await reader.OpenEntryStreamAsync(CancellationToken.None); -#endif + var stream = await reader.OpenEntryStreamAsync(CancellationToken.None); + await using var scope = stream.DisposeAsyncScope().ConfigureAwait(false); await stream.CopyToAsync(Stream.Null, Constants.BufferSize, CancellationToken.None); if (!readAll) { @@ -87,11 +84,8 @@ public class SevenZipLifetimeTests : TestBase // Disposing a borrowed reader must not prevent another extraction. var entry = await archive.EntriesAsync.Where(entry => !entry.IsDirectory).FirstAsync(); -#if LEGACY_DOTNET - using var entryStream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#else - await using var entryStream = await entry.OpenEntryStreamAsync(CancellationToken.None); -#endif + var entryStream = await entry.OpenEntryStreamAsync(CancellationToken.None); + await using var scope = entryStream.DisposeAsyncScope().ConfigureAwait(false); await entryStream.CopyToAsync( Stream.Null, Constants.BufferSize, diff --git a/tests/SharpCompress.Test/Zip/ZipEncryptedDataDescriptorTests.cs b/tests/SharpCompress.Test/Zip/ZipEncryptedDataDescriptorTests.cs index 0eb13ab2..d6be341f 100644 --- a/tests/SharpCompress.Test/Zip/ZipEncryptedDataDescriptorTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipEncryptedDataDescriptorTests.cs @@ -188,11 +188,8 @@ public class ZipEncryptedDataDescriptorTests : TestBase continue; } -#if LEGACY_DOTNET - using var entryStream = await entry.OpenEntryStreamAsync(); -#else - await using var entryStream = await entry.OpenEntryStreamAsync(); -#endif + var entryStream = await entry.OpenEntryStreamAsync(); + await using var scope = entryStream.DisposeAsyncScope().ConfigureAwait(false); using var output = new MemoryStream(); await entryStream.CopyToAsync(output); contents.Add(entry.Key!, output.ToArray()); @@ -335,11 +332,8 @@ public class ZipEncryptedDataDescriptorTests : TestBase private static async ValueTask ReadAsync(IAsyncReader reader) { -#if LEGACY_DOTNET - using var entryStream = await reader.OpenEntryStreamAsync(); -#else - await using var entryStream = await reader.OpenEntryStreamAsync(); -#endif + var entryStream = await reader.OpenEntryStreamAsync(); + await using var scope = entryStream.DisposeAsyncScope().ConfigureAwait(false); using var output = new MemoryStream(); await entryStream.CopyToAsync(output); return output.ToArray(); diff --git a/tests/SharpCompress.Test/Zip/ZipWinzipAesReaderTests.cs b/tests/SharpCompress.Test/Zip/ZipWinzipAesReaderTests.cs index c6c7634f..ede8fe11 100644 --- a/tests/SharpCompress.Test/Zip/ZipWinzipAesReaderTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipWinzipAesReaderTests.cs @@ -200,11 +200,8 @@ public class ZipWinzipAesReaderTests : TestBase continue; } -#if LEGACY_DOTNET - using var entryStream = await entry.OpenEntryStreamAsync(); -#else - await using var entryStream = await entry.OpenEntryStreamAsync(); -#endif + var entryStream = await entry.OpenEntryStreamAsync(); + await using var scope = entryStream.DisposeAsyncScope().ConfigureAwait(false); Assert.Equal(expected, await ReadInChunksAsync(entryStream, bufferSize)); } } @@ -303,11 +300,8 @@ public class ZipWinzipAesReaderTests : TestBase private static async ValueTask ReadAsync(IAsyncReader reader) { -#if LEGACY_DOTNET - using var entryStream = await reader.OpenEntryStreamAsync(); -#else - await using var entryStream = await reader.OpenEntryStreamAsync(); -#endif + var entryStream = await reader.OpenEntryStreamAsync(); + await using var scope = entryStream.DisposeAsyncScope().ConfigureAwait(false); using var output = new MemoryStream(); await entryStream.CopyToAsync(output); return output.ToArray(); From 1414e17f6ff428240c389c809c6f43c5e7338563 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 19 Oct 2026 12:01:46 +0100 Subject: [PATCH 2/3] Avoid copying pooled 7z headers when computing async CRCs --- .../Writers/SevenZip/SevenZipWriter.Async.cs | 30 +++++------ .../SevenZip/SevenZipWriterAsyncTests.cs | 53 +++++++++++++++++++ 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs index 4d61dfad..c1cbaf6f 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs @@ -103,21 +103,18 @@ public partial class SevenZipWriter // Write the encoded header to the output var headerStartPos = output.Position; encodedHeaderStream.Position = 0; - await encodedHeaderStream.CopyToAsync(output).ConfigureAwait(false); - - // Compute CRC of the encoded header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - encodedHeaderStream.GetBuffer().AsSpan(0, (int)encodedHeaderStream.Length) - ); + // Compute the CRC during the copy without materializing the pooled header. + using var crcStream = new Crc32Stream(output); + await encodedHeaderStream + .CopyToAsync(crcStream, Constants.BufferSize, CancellationToken.None) + .ConfigureAwait(false); // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize); var nextHeaderSize = (ulong)encodedHeaderStream.Length; await SevenZipSignatureHeaderWriter - .WriteFinalAsync(output, nextHeaderOffset, nextHeaderSize, headerCrc) + .WriteFinalAsync(output, nextHeaderOffset, nextHeaderSize, crcStream.Crc) .ConfigureAwait(false); // Seek to end @@ -134,21 +131,18 @@ public partial class SevenZipWriter // Write raw header directly var headerStartPos = output.Position; rawHeaderStream.Position = 0; - await rawHeaderStream.CopyToAsync(output).ConfigureAwait(false); - - // Compute CRC of the raw header - var headerCrc = Crc32Stream.Compute( - Crc32Stream.DEFAULT_POLYNOMIAL, - Crc32Stream.DEFAULT_SEED, - rawHeaderStream.GetBuffer().AsSpan(0, (int)rawHeaderStream.Length) - ); + // Compute the CRC during the copy without materializing the pooled header. + using var crcStream = new Crc32Stream(output); + await rawHeaderStream + .CopyToAsync(crcStream, Constants.BufferSize, CancellationToken.None) + .ConfigureAwait(false); // Back-patch signature header var nextHeaderOffset = (ulong)(headerStartPos - SevenZipSignatureHeaderWriter.HeaderSize); var nextHeaderSize = (ulong)rawHeaderStream.Length; await SevenZipSignatureHeaderWriter - .WriteFinalAsync(output, nextHeaderOffset, nextHeaderSize, headerCrc) + .WriteFinalAsync(output, nextHeaderOffset, nextHeaderSize, crcStream.Crc) .ConfigureAwait(false); // Seek to end diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipWriterAsyncTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipWriterAsyncTests.cs index e61b77d7..9ebb5c34 100644 --- a/tests/SharpCompress.Test/SevenZip/SevenZipWriterAsyncTests.cs +++ b/tests/SharpCompress.Test/SevenZip/SevenZipWriterAsyncTests.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers.Binary; using System.IO; using System.Linq; using System.Text; @@ -6,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Archives.SevenZip; using SharpCompress.Common; +using SharpCompress.Crypto; using SharpCompress.Test.Mocks; using SharpCompress.Writers; using SharpCompress.Writers.SevenZip; @@ -15,6 +17,57 @@ namespace SharpCompress.Test.SevenZip; public class SevenZipWriterAsyncTests : TestBase { + [Theory] + [InlineData(false, 1)] + [InlineData(true, 1)] + [InlineData(false, 256)] + [InlineData(true, 256)] + public async ValueTask SevenZipWriter_Async_HeaderCrc_MatchesWrittenBytes( + bool compressHeader, + int directoryCount + ) + { + using var output = new MemoryStream(); + await using ( + var writer = new SevenZipWriter( + new AsyncOnlyStream(output), + new SevenZipWriterOptions { CompressHeader = compressHeader } + ) + ) + { + // Large names make the raw header span multiple pooled blocks. + for (var i = 0; i < directoryCount; i++) + { + await writer.WriteDirectoryAsync( + $"dir{i}-{new string('x', 512)}", + null, + CancellationToken.None + ); + } + } + + Assert.True(output.CanWrite); + var bytes = output.ToArray(); + var headerOffset = BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(12)); + var headerSize = BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(20)); + var headerCrc = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(28)); + Assert.Equal((ulong)bytes.Length, 32 + headerOffset + headerSize); + Assert.Equal( + Crc32Stream.Compute( + Crc32Stream.DEFAULT_POLYNOMIAL, + Crc32Stream.DEFAULT_SEED, + bytes.AsSpan(checked(32 + (int)headerOffset), checked((int)headerSize)) + ), + headerCrc + ); + + output.Position = 0; + await using var archive = await SevenZipArchive.OpenAsyncArchive(output); + var entries = await archive.EntriesAsync.ToListAsync(); + Assert.Equal(directoryCount, entries.Count); + Assert.All(entries, entry => Assert.True(entry.IsDirectory)); + } + [Fact] public async ValueTask SevenZipWriter_Async_SingleFile_RoundTrip() { From 78006f94f72146ccb096442f11ebc5f6f43c26a1 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 19 Oct 2026 12:07:29 +0100 Subject: [PATCH 3/3] More pooled or readonly memory stream usage --- .../Compressors/LZMA/Lzma2EncoderStream.cs | 58 ++++++----- .../Compressors/LZMA/LzmaStream.Async.cs | 3 +- .../Compressors/PPMd/PpmdStream.cs | 8 +- src/SharpCompress/IO/ReadOnlyMemoryStream.cs | 99 +++++++++++++++++++ .../Benchmarks/GZipBenchmarks.cs | 17 ++++ .../Benchmarks/TarBenchmarks.cs | 33 +++++++ .../Benchmarks/ZipBenchmarks.cs | 33 +++++++ .../Streams/Lzma2EncoderBufferTests.cs | 61 ++++++++++++ .../Streams/MemoryEncoderWriteTests.cs | 67 +++++++++++++ .../Streams/ReadOnlyMemoryStreamTests.cs | 43 ++++++++ 10 files changed, 390 insertions(+), 32 deletions(-) create mode 100644 src/SharpCompress/IO/ReadOnlyMemoryStream.cs create mode 100644 tests/SharpCompress.Test/Streams/Lzma2EncoderBufferTests.cs create mode 100644 tests/SharpCompress.Test/Streams/MemoryEncoderWriteTests.cs create mode 100644 tests/SharpCompress.Test/Streams/ReadOnlyMemoryStreamTests.cs diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 2790b7c5..e75e787f 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.Common; using SharpCompress.IO; namespace SharpCompress.Compressors.LZMA; @@ -259,7 +260,7 @@ internal sealed class Lzma2EncoderStream : Stream _bufferPosition = 0; // Try compressing the data - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -271,13 +272,14 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; // Check if compressed output fits in a single chunk and is actually smaller if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length ) { - WriteCompressedChunk(uncompressedData.Length, compressed.Buffer, compressed.Length); + WriteCompressedChunk(uncompressedData.Length, compressed, (int)compressed.Length); } else { @@ -296,7 +298,7 @@ internal sealed class Lzma2EncoderStream : Stream var uncompressedData = _buffer.AsMemory(0, uncompressedSize); _bufferPosition = 0; - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -308,6 +310,7 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length @@ -315,8 +318,8 @@ internal sealed class Lzma2EncoderStream : Stream { await WriteCompressedChunkAsync( uncompressedData.Length, - compressed.Buffer, - compressed.Length, + compressed, + (int)compressed.Length, cancellationToken ) .ConfigureAwait(false); @@ -328,23 +331,29 @@ internal sealed class Lzma2EncoderStream : Stream } } - private (byte[] Buffer, int Length) CompressBlock(byte[] data, int length) + private PooledMemoryStream CompressBlock(byte[] data, int length) { using var inputMs = new MemoryStream(data, 0, length, writable: false); - using var outputMs = new PooledMemoryStream(); - - _encoder.Code(inputMs, outputMs, length, -1, null); - - var fullCompressed = outputMs.ToArray(); - - // The LZMA range encoder flush writes trailing bytes the decoder doesn't consume. - // Trial-decode to find the exact byte count the decoder needs, so the LZMA2 - // chunk header reports a compressed size that matches what the decoder reads. - var consumed = FindConsumedBytes(fullCompressed, fullCompressed.Length, length); - return (fullCompressed, consumed); + var outputMs = new PooledMemoryStream(); + try + { + _encoder.Code(inputMs, outputMs, length, -1, null); + // Trial-decode to exclude trailing range-encoder bytes from the chunk size. + outputMs.Position = 0; + var consumed = FindConsumedBytes(outputMs, length); + outputMs.SetLength(consumed); + outputMs.Position = 0; + // The caller owns the pooled output until the chunk write completes. + return outputMs; + } + catch + { + outputMs.Dispose(); + throw; + } } - private int FindConsumedBytes(byte[] compressedData, int compressedSize, int uncompressedSize) + private int FindConsumedBytes(PooledMemoryStream input, int uncompressedSize) { // Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32] Span props = stackalloc byte[5]; @@ -357,8 +366,7 @@ internal sealed class Lzma2EncoderStream : Stream using var decoder = new Decoder(); decoder.SetDecoderProperties(props); - using var input = new MemoryStream(compressedData, 0, compressedSize, writable: false); - decoder.Code(input, Stream.Null, compressedSize, uncompressedSize, null); + decoder.Code(input, Stream.Null, input.Length, uncompressedSize, null); return (int)input.Position; } @@ -369,7 +377,7 @@ internal sealed class Lzma2EncoderStream : Stream /// private void WriteCompressedChunk( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize ) { @@ -393,12 +401,12 @@ internal sealed class Lzma2EncoderStream : Stream // 0xE0 (>= 0xC0) requires properties byte _output.WriteByte(_lzmaPropertiesByte); - _output.Write(compressedData, 0, compressedSize); + compressedData.WriteTo(_output); } private async ValueTask WriteCompressedChunkAsync( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize, CancellationToken cancellationToken = default ) @@ -423,8 +431,8 @@ internal sealed class Lzma2EncoderStream : Stream { ArrayPool.Shared.Return(header); } - await _output - .WriteAsync(compressedData, 0, compressedSize, cancellationToken) + await compressedData + .CopyToAsync(_output, Constants.BufferSize, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 653e6920..c64d75b5 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -465,8 +465,9 @@ public partial class LzmaStream cancellationToken.ThrowIfCancellationRequested(); if (_encoder != null) { + using var input = new ReadOnlyMemoryStream(buffer); _position = await _encoder - .CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken) + .CodeAsync(input, false, cancellationToken) .ConfigureAwait(false); } } diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index f04af7fb..23d296a5 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -412,13 +412,9 @@ public class PpmdStream : Stream, IAsyncDisposable cancellationToken.ThrowIfCancellationRequested(); if (_compress) { + using var input = new ReadOnlyMemoryStream(buffer); await _model - .EncodeBlockAsync( - _stream, - new MemoryStream(buffer.ToArray()), - false, - cancellationToken - ) + .EncodeBlockAsync(_stream, input, false, cancellationToken) .ConfigureAwait(false); } } diff --git a/src/SharpCompress/IO/ReadOnlyMemoryStream.cs b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs new file mode 100644 index 00000000..4501f24c --- /dev/null +++ b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs @@ -0,0 +1,99 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +// Borrows memory for the lifetime of an encoder call; never copies or owns the backing storage. +internal sealed class ReadOnlyMemoryStream(ReadOnlyMemory memory) : Stream +{ + private ReadOnlyMemory _memory = memory; + private int _position; + private bool _disposed; + + public override bool CanRead => !_disposed; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => + !_disposed + ? _memory.Length + : throw new ObjectDisposedException(nameof(ReadOnlyMemoryStream)); + public override long Position + { + get => + !_disposed + ? _position + : throw new ObjectDisposedException(nameof(ReadOnlyMemoryStream)); + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + ThrowHelper.ThrowIfNull(buffer, nameof(buffer)); + if (offset < 0 || count < 0 || offset > buffer.Length - count) + { + throw new ArgumentOutOfRangeException(nameof(count), "Invalid buffer range."); + } + return ReadCore(buffer.AsSpan(offset, count)); + } + + private int ReadCore(Span buffer) + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ReadOnlyMemoryStream)); + } + var count = Math.Min(buffer.Length, _memory.Length - _position); + _memory.Span.Slice(_position, count).CopyTo(buffer); + _position += count; + return count; + } + + public override int ReadByte() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ReadOnlyMemoryStream)); + } + return _position < _memory.Length ? _memory.Span[_position++] : -1; + } + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) => + cancellationToken.IsCancellationRequested + ? Task.FromCanceled(cancellationToken) + : Task.FromResult(Read(buffer, offset, count)); + +#if !LEGACY_DOTNET + public override int Read(Span buffer) => ReadCore(buffer); + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) => + cancellationToken.IsCancellationRequested + ? ValueTask.FromCanceled(cancellationToken) + : ValueTask.FromResult(ReadCore(buffer.Span)); +#endif + + public override void Flush() { } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + _disposed = true; + _memory = default; + base.Dispose(disposing); + } +} diff --git a/tests/SharpCompress.Performance/Benchmarks/GZipBenchmarks.cs b/tests/SharpCompress.Performance/Benchmarks/GZipBenchmarks.cs index e9804009..4e0c63e1 100644 --- a/tests/SharpCompress.Performance/Benchmarks/GZipBenchmarks.cs +++ b/tests/SharpCompress.Performance/Benchmarks/GZipBenchmarks.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using SharpCompress.Compressors; using SharpCompress.Compressors.Deflate; +using SharpCompress.IO; namespace SharpCompress.Performance.Benchmarks; @@ -45,6 +46,22 @@ public class GZipBenchmarks await gzipStream.WriteAsync(_sourceData, 0, _sourceData.Length).ConfigureAwait(false); } + [Benchmark(Description = "GZip: Compress 100KB (Pooled)")] + public void GZipCompressPooled() + { + using var outputStream = new PooledMemoryStream(); + using var gzipStream = new GZipStream(outputStream, CompressionMode.Compress); + gzipStream.Write(_sourceData, 0, _sourceData.Length); + } + + [Benchmark(Description = "GZip: Compress 100KB (Async, Pooled)")] + public async Task GZipCompressAsyncPooled() + { + using var outputStream = new PooledMemoryStream(); + using var gzipStream = new GZipStream(outputStream, CompressionMode.Compress); + await gzipStream.WriteAsync(_sourceData, 0, _sourceData.Length).ConfigureAwait(false); + } + [Benchmark(Description = "GZip: Decompress 100KB")] public void GZipDecompress() { diff --git a/tests/SharpCompress.Performance/Benchmarks/TarBenchmarks.cs b/tests/SharpCompress.Performance/Benchmarks/TarBenchmarks.cs index 7a7cbadd..8018dc55 100644 --- a/tests/SharpCompress.Performance/Benchmarks/TarBenchmarks.cs +++ b/tests/SharpCompress.Performance/Benchmarks/TarBenchmarks.cs @@ -7,6 +7,7 @@ using SharpCompress.Archives; using SharpCompress.Archives.Tar; using SharpCompress.Common; using SharpCompress.Compressors; +using SharpCompress.IO; using SharpCompress.Providers; using SharpCompress.Providers.System; using SharpCompress.Readers; @@ -127,6 +128,38 @@ public class TarBenchmarks : ArchiveBenchmarkBase } } + [Benchmark(Description = "Tar: Create archive with small files (Pooled)")] + public void TarCreateSmallFilesPooled() + { + using var outputStream = new PooledMemoryStream(); + using var writer = WriterFactory.OpenWriter( + outputStream, + ArchiveType.Tar, + new WriterOptions(CompressionType.None) { LeaveStreamOpen = true } + ); + for (var i = 0; i < 10; i++) + { + using var entryStream = new MemoryStream(new byte[1024]); + writer.Write($"file{i}.txt", entryStream); + } + } + + [Benchmark(Description = "Tar: Create archive with small files (Async, Pooled)")] + public async Task TarCreateSmallFilesAsyncPooled() + { + using var outputStream = new PooledMemoryStream(); + await using var writer = await WriterFactory.OpenAsyncWriter( + outputStream, + ArchiveType.Tar, + new WriterOptions(CompressionType.None) { LeaveStreamOpen = true } + ); + for (var i = 0; i < 10; i++) + { + using var entryStream = new MemoryStream(new byte[1024]); + await writer.WriteAsync($"file{i}.txt", entryStream).ConfigureAwait(false); + } + } + [Benchmark(Description = "Tar: Create archive with small files")] public void TarCreateSmallFiles() { diff --git a/tests/SharpCompress.Performance/Benchmarks/ZipBenchmarks.cs b/tests/SharpCompress.Performance/Benchmarks/ZipBenchmarks.cs index c5b3e090..252177d3 100644 --- a/tests/SharpCompress.Performance/Benchmarks/ZipBenchmarks.cs +++ b/tests/SharpCompress.Performance/Benchmarks/ZipBenchmarks.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Attributes; using SharpCompress.Archives.Zip; using SharpCompress.Common; using SharpCompress.Compressors; +using SharpCompress.IO; using SharpCompress.Providers; using SharpCompress.Providers.System; using SharpCompress.Readers; @@ -114,6 +115,38 @@ public class ZipBenchmarks : ArchiveBenchmarkBase } } + [Benchmark(Description = "Zip: Create archive with small files (Pooled)")] + public void ZipCreateSmallFilesPooled() + { + using var outputStream = new PooledMemoryStream(); + using var writer = WriterFactory.OpenWriter( + outputStream, + ArchiveType.Zip, + new WriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true } + ); + for (var i = 0; i < 10; i++) + { + using var entryStream = new MemoryStream(new byte[1024]); + writer.Write($"file{i}.txt", entryStream); + } + } + + [Benchmark(Description = "Zip: Create archive with small files (Async, Pooled)")] + public async Task ZipCreateSmallFilesAsyncPooled() + { + using var outputStream = new PooledMemoryStream(); + await using var writer = await WriterFactory.OpenAsyncWriter( + outputStream, + ArchiveType.Zip, + new WriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true } + ); + for (var i = 0; i < 10; i++) + { + using var entryStream = new MemoryStream(new byte[1024]); + await writer.WriteAsync($"file{i}.txt", entryStream).ConfigureAwait(false); + } + } + [Benchmark(Description = "Zip: Create archive with small files")] public void ZipCreateSmallFiles() { diff --git a/tests/SharpCompress.Test/Streams/Lzma2EncoderBufferTests.cs b/tests/SharpCompress.Test/Streams/Lzma2EncoderBufferTests.cs new file mode 100644 index 00000000..0fb86ca5 --- /dev/null +++ b/tests/SharpCompress.Test/Streams/Lzma2EncoderBufferTests.cs @@ -0,0 +1,61 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Compressors.LZMA; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class Lzma2EncoderBufferTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Pooled_Compressed_Output_RoundTrips(bool useAsync, bool incompressible) + { + var data = new byte[150000]; + if (incompressible) + { + new Random(42).NextBytes(data); + } + using var packed = new MemoryStream(); + using (var encoder = new Lzma2EncoderStream(packed, 1 << 16, 32)) + { + if (useAsync) + { + await encoder.WriteAsync(data, 0, data.Length, CancellationToken.None); + await encoder.DisposeAsync(); + } + else + { + encoder.Write(data, 0, data.Length); + } + } + packed.Position = 0; + using var decoded = new MemoryStream(); + if (useAsync) + { + await using var decoder = await LzmaStream.CreateAsync( + new byte[] { 8 }, + packed, + packed.Length, + data.Length + ); + await decoder.CopyToAsync(decoded, 81920, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create( + new byte[] { 8 }, + packed, + packed.Length, + data.Length + ); + decoder.CopyTo(decoded); + } + Assert.Equal(data, decoded.ToArray()); + } +} diff --git a/tests/SharpCompress.Test/Streams/MemoryEncoderWriteTests.cs b/tests/SharpCompress.Test/Streams/MemoryEncoderWriteTests.cs new file mode 100644 index 00000000..cf58dcc2 --- /dev/null +++ b/tests/SharpCompress.Test/Streams/MemoryEncoderWriteTests.cs @@ -0,0 +1,67 @@ +#if !LEGACY_DOTNET +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.PPMd; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class MemoryEncoderWriteTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Memory_Writes_Match_Array_Writes(bool ppmd) + { + var data = new byte[20000]; + new Random(42).NextBytes(data); + using var memory = new TestMemoryManager(data); + using var expected = new MemoryStream(); + using var actual = new MemoryStream(); + await Encode(expected, false); + await Encode(actual, true); + Assert.Equal(expected.ToArray(), actual.ToArray()); + Assert.Equal(data, memory.GetSpan().ToArray()); + + async Task Encode(Stream output, bool useMemory) + { + await using Stream encoder = ppmd + ? PpmdStream.Create(new PpmdProperties(), output, true) + : LzmaStream.Create(new LzmaEncoderProperties(true, 4096), false, output); + for (var offset = 1; offset < data.Length - 1; ) + { + var count = Math.Min(137, data.Length - 1 - offset); + if (useMemory) + { + await encoder.WriteAsync( + memory.Memory.Slice(offset, count), + CancellationToken.None + ); + } + else + { + await encoder.WriteAsync(data, offset, count, CancellationToken.None); + } + offset += count; + } + await encoder.WriteAsync(ReadOnlyMemory.Empty, CancellationToken.None); + } + } + + // Exercises memory that cannot be unwrapped with MemoryMarshal.TryGetArray. + private sealed class TestMemoryManager(byte[] data) : MemoryManager + { + public override Span GetSpan() => data; + + public override MemoryHandle Pin(int elementIndex = 0) => throw new NotSupportedException(); + + public override void Unpin() { } + + protected override void Dispose(bool disposing) { } + } +} +#endif diff --git a/tests/SharpCompress.Test/Streams/ReadOnlyMemoryStreamTests.cs b/tests/SharpCompress.Test/Streams/ReadOnlyMemoryStreamTests.cs new file mode 100644 index 00000000..b49d0039 --- /dev/null +++ b/tests/SharpCompress.Test/Streams/ReadOnlyMemoryStreamTests.cs @@ -0,0 +1,43 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class ReadOnlyMemoryStreamTests +{ + [Fact] + public async Task Reads_Borrowed_Slice_Without_Copying() + { + var data = new byte[] { 99, 1, 2, 3, 99 }; + using var stream = new ReadOnlyMemoryStream(data.AsMemory(1, 3)); + data[1] = 4; + Assert.Equal(4, stream.ReadByte()); + var buffer = new byte[4]; + Assert.Equal(2, await stream.ReadAsync(buffer, 1, 3, CancellationToken.None)); + Assert.Equal(new byte[] { 0, 2, 3, 0 }, buffer); + Assert.Equal(3, stream.Position); + Assert.Equal(0, stream.Read(buffer, 0, buffer.Length)); + Assert.Equal(-1, stream.ReadByte()); + } + + [Fact] + public async Task Cancellation_And_Disposal_Do_Not_Consume_Memory() + { + var data = new byte[] { 1, 2, 3 }; + var stream = new ReadOnlyMemoryStream(data); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => + stream.ReadAsync(new byte[3], 0, 3, cts.Token) + ); + Assert.Equal(0, stream.Position); + stream.Dispose(); + Assert.False(stream.CanRead); + Assert.Throws(() => stream.ReadByte()); + Assert.Equal(new byte[] { 1, 2, 3 }, data); + } +}