From bc1ade2bbc424beae2a74a8c86875ed84b273de2 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 15 Oct 2026 10:44:50 +0100 Subject: [PATCH 1/7] Reject truncated LZMA2 headers and prevent stalled decoding --- .../Compressors/LZMA/LzmaStream.Async.cs | 18 ++ .../Compressors/LZMA/LzmaStream.cs | 37 +++-- .../Streams/Lzma2TruncationTests.cs | 156 ++++++++++++++++++ 3 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index c89fb923..24b28922 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -201,6 +201,11 @@ public partial class LzmaStream } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -224,6 +229,10 @@ public partial class LzmaStream } var read = _outWindow.Read(buffer, offset, toProcess); + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; @@ -317,6 +326,11 @@ public partial class LzmaStream } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -340,6 +354,10 @@ public partial class LzmaStream } var read = _outWindow.Read(buffer, offset, toProcess); + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index f45b33a3..e4454771 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -266,6 +266,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -283,6 +288,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } var read = _outWindow.Read(buffer, offset, toProcess); + // Unknown-size raw LZMA can end without output; otherwise no progress is corruption. + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; @@ -416,10 +426,21 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable return value; } + private int ReadChunkHeaderByte() + { + var value = _inputStream!.ReadByte(); + // EOF is not an LZMA2 end marker and must never enter chunk-size arithmetic. + if (value < 0) + { + throw new DataErrorException(); + } + _inputPosition++; + return value; + } + private void DecodeChunkHeader() { - var control = _inputStream!.ReadByte(); - _inputPosition++; + var control = ReadChunkHeaderByte(); if (control == 0x00) { @@ -448,17 +469,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable _uncompressedChunk = false; _availableBytes = (control & 0x1F) << 16; - _availableBytes += (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _availableBytes += (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; - _rangeDecoderLimit = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _rangeDecoderLimit = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; if (control >= 0xC0) { _needProps = false; - Properties[0] = (byte)_inputStream.ReadByte(); - _inputPosition++; + Properties[0] = (byte)ReadChunkHeaderByte(); _decoder = new Decoder(); _decoder.SetDecoderProperties(Properties); @@ -482,8 +500,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable else { _uncompressedChunk = true; - _availableBytes = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _availableBytes = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; } } diff --git a/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs new file mode 100644 index 00000000..4fa16d41 --- /dev/null +++ b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.Xz; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class Lzma2TruncationTests +{ + private const string TruncatedXzHex = + "FD377A585A0000016922DE360200210100000000372797D6010011616C7068610A616C7068610A616C7068610A"; + private const string MutatedSevenZipBase64 = + "N3q8ryccAATrtqxIDgAAAAAAAABaAAAAAAAAAKyqq98BAApoZWxsbyB3b3JsZAEEBgABCQ4ABwsBAAEhIQEADAwACAoBhRFKDQAABQEZDAAAAAAAAAAAAAAAABETAHQAZQBzAHQALgB0AHgAdAAAABkAFAoBAI6gjYxMV90BFQYBACAAAAAAAA=="; + + public static IEnumerable PartialHeaders() + { + // Start with a complete dictionary-reset chunk so a missing control byte + // cannot be rejected merely because the dictionary still needs resetting. + byte[] prefix = [0x01, 0x00, 0x00, 0x58]; + yield return [prefix]; + byte[][] headers = + [ + [0x01, 0x00, 0x00], + [0x02, 0x00, 0x00], + [0xE0, 0x00, 0x00, 0x00, 0x04, 0x00], + ]; + foreach (var header in headers) + { + for (var length = 1; length < header.Length; length++) + { + var data = new byte[prefix.Length + length]; + prefix.CopyTo(data, 0); + Array.Copy(header, 0, data, prefix.Length, length); + yield return [data]; + } + } + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public void Lzma2_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + using var decoder = LzmaStream.Create([0x00], input); + Assert.ThrowsAny(() => decoder.CopyTo(Stream.Null)); + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public void Lzma2_ReadByte_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + using var decoder = LzmaStream.Create([0x00], input); + Assert.Equal(0x58, decoder.ReadByte()); + Assert.ThrowsAny(() => decoder.ReadByte()); + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public async Task Lzma2_Async_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + await using var decoder = await LzmaStream.CreateAsync([0x00], input, -1, -1); + var buffer = new byte[64]; + await Assert.ThrowsAnyAsync(async () => + await decoder.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None) + ); +#if !LEGACY_DOTNET + // Use a fresh decoder because parser state is undefined after a failed read. + using var memoryInput = new MemoryStream(data); + await using var memoryDecoder = await LzmaStream.CreateAsync([0x00], memoryInput, -1, -1); + await Assert.ThrowsAnyAsync(async () => + await memoryDecoder.ReadAsync(buffer.AsMemory(), CancellationToken.None) + ); +#endif + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Xz_Rejects_Missing_Lzma2_End_Marker(bool useAsync) + { + var data = new byte[TruncatedXzHex.Length / 2]; + for (var i = 0; i < data.Length; i++) + { + data[i] = Convert.ToByte(TruncatedXzHex.Substring(i * 2, 2), 16); + } + using var input = new MemoryStream(data); + await using var decoder = new XZStream(input); + if (useAsync) + { + await Assert.ThrowsAnyAsync(() => + decoder.CopyToAsync(Stream.Null, 81920, CancellationToken.None) + ); + } + else + { + Assert.ThrowsAny(() => decoder.CopyTo(Stream.Null)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SevenZip_Rejects_Missing_Lzma2_End_Marker(bool useAsync) + { + using var input = new MemoryStream(Convert.FromBase64String(MutatedSevenZipBase64)); + if (useAsync) + { + await using var archive = await ArchiveFactory.OpenAsyncArchive(input); + await foreach (var entry in archive.EntriesAsync) + { + await using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); + await Assert.ThrowsAnyAsync(() => + stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None) + ); + } + } + else + { + using var archive = ArchiveFactory.OpenArchive(input); + foreach (var entry in archive.Entries) + { + using var stream = entry.OpenEntryStream(); + Assert.ThrowsAny(() => stream.CopyTo(Stream.Null)); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma2_Accepts_End_Marker_After_Uncompressed_Chunks(bool useAsync) + { + byte[] data = [0x01, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x59, 0x00]; + using var input = new MemoryStream(data); + using var output = new MemoryStream(); + if (useAsync) + { + await using var decoder = await LzmaStream.CreateAsync([0x00], input, data.Length, 2); + await decoder.CopyToAsync(output, 1, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create([0x00], input, data.Length, 2); + decoder.CopyTo(output, 1); + } + Assert.Equal(new byte[] { 0x58, 0x59 }, output.ToArray()); + } +} From 9b0be6f9847cca97dfd593bb058c4ebe67a7dd70 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 16 Oct 2026 15:33:10 +0100 Subject: [PATCH 2/7] Skip ZIP64 extensible data without size-driven allocations --- docs/USAGE.md | 3 + .../Headers/Zip64DirectoryEndHeader.Async.cs | 19 ++++-- .../Zip/Headers/Zip64DirectoryEndHeader.cs | 36 +++++++--- .../Zip/Zip64AllocationTests.cs | 68 +++++++++++++++++++ 4 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs diff --git a/docs/USAGE.md b/docs/USAGE.md index 9666d43f..18249a87 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -472,3 +472,6 @@ await archive.WriteToDirectoryAsync( - Support for cancellation via CancellationToken - Better resource utilization in async/await contexts - Compatible with modern .NET async patterns +## Malformed input and memory use + +ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size. diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs index 9e510688..df9e7cbe 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Threading.Tasks; using SharpCompress.IO; @@ -8,7 +9,9 @@ internal partial class Zip64DirectoryEndHeader { internal override async ValueTask Read(AsyncBinaryReader reader) { - SizeOfDirectoryEndRecord = (long)await reader.ReadUInt64Async().ConfigureAwait(false); + SizeOfDirectoryEndRecord = ValidateRecordSize( + await reader.ReadUInt64Async().ConfigureAwait(false) + ); VersionMadeBy = await reader.ReadUInt16Async().ConfigureAwait(false); VersionNeededToExtract = await reader.ReadUInt16Async().ConfigureAwait(false); VolumeNumber = await reader.ReadUInt32Async().ConfigureAwait(false); @@ -18,10 +21,14 @@ internal partial class Zip64DirectoryEndHeader DirectorySize = (long)await reader.ReadUInt64Async().ConfigureAwait(false); DirectoryStartOffsetRelativeToDisk = (long) await reader.ReadUInt64Async().ConfigureAwait(false); - var size = (int)( - SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS - ); - DataSector = new byte[size]; - await reader.ReadBytesAsync(DataSector, 0, size).ConfigureAwait(false); + var remaining = + SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; + var buffer = new byte[4096]; + while (remaining > 0) + { + var size = (int)Math.Min(buffer.Length, remaining); + await reader.ReadBytesAsync(buffer, 0, size).ConfigureAwait(false); + remaining -= size; + } } } diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs index 3933b2e0..96d01637 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs @@ -1,3 +1,4 @@ +using System; using System.IO; namespace SharpCompress.Common.Zip.Headers; @@ -9,7 +10,7 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader internal override void Read(BinaryReader reader) { - SizeOfDirectoryEndRecord = (long)reader.ReadUInt64(); + SizeOfDirectoryEndRecord = ValidateRecordSize(reader.ReadUInt64()); VersionMadeBy = reader.ReadUInt16(); VersionNeededToExtract = reader.ReadUInt16(); VolumeNumber = reader.ReadUInt32(); @@ -18,12 +19,31 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader TotalNumberOfEntries = (long)reader.ReadUInt64(); DirectorySize = (long)reader.ReadUInt64(); DirectoryStartOffsetRelativeToDisk = (long)reader.ReadUInt64(); - DataSector = reader.ReadBytes( - (int)( - SizeOfDirectoryEndRecord - - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS - ) - ); + // Extensible data is not interpreted. Consume it without allocating from its declared size. + var remaining = + SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; + var buffer = new byte[4096]; + while (remaining > 0) + { + var read = reader.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); + if (read == 0) + { + throw new IncompleteArchiveException("Truncated ZIP64 extensible data."); + } + remaining -= read; + } + } + + private static long ValidateRecordSize(ulong size) + { + if ( + size < SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS + || size > long.MaxValue + ) + { + throw new InvalidFormatException("Invalid ZIP64 end record size."); + } + return (long)size; } private const int SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS = 44; @@ -45,6 +65,4 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader public long DirectorySize { get; private set; } public long DirectoryStartOffsetRelativeToDisk { get; private set; } - - public byte[]? DataSector { get; private set; } } diff --git a/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs b/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs new file mode 100644 index 00000000..48ed6024 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Common.Zip.Headers; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class Zip64AllocationTests +{ + [Theory] + [InlineData(0UL)] + [InlineData(43UL)] + [InlineData(ulong.MaxValue)] + public async Task Zip64_Rejects_Invalid_Record_Size(ulong size) + { + using var stream = CreateRecord(size, 0); + Assert.Throws(() => + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)) + ); + stream.Position = 0; + await Assert.ThrowsAsync(async () => + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)) + ); + } + + [Theory] + [InlineData(44UL, 0)] + [InlineData(10044UL, 10000)] + public async Task Zip64_Skips_Extensible_Data_Exactly(ulong size, int payloadSize) + { + using var stream = CreateRecord(size, payloadSize); + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)); + Assert.Equal(stream.Length, stream.Position); + stream.Position = 0; + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)); + Assert.Equal(stream.Length, stream.Position); + } + + [Theory] + [InlineData(2147483691UL)] + [InlineData(4294967340UL)] + public async Task Zip64_Rejects_Huge_Truncated_Extensible_Data(ulong size) + { + using var stream = CreateRecord(size, 1); + Assert.ThrowsAny(() => + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)) + ); + stream.Position = 0; + await Assert.ThrowsAnyAsync(async () => + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)) + ); + } + + private static MemoryStream CreateRecord(ulong size, int payloadSize) + { + var stream = new MemoryStream(); + using (var writer = new BinaryWriter(stream, System.Text.Encoding.UTF8, true)) + { + writer.Write(size); + writer.Write(new byte[44 + payloadSize]); + } + stream.Position = 0; + return stream; + } +} From 0860da8bc5c441ba89f8e5a444c8eb252740025e Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 16 Oct 2026 15:34:07 +0100 Subject: [PATCH 3/7] Buffer ARC LZW input incrementally and reject truncated entries --- docs/USAGE.md | 2 + .../Common/Arc/ArcFilePart.Async.cs | 6 ++ src/SharpCompress/Common/Arc/ArcFilePart.cs | 6 ++ .../Compressors/ArcLzw/ArcLzwStream.Async.cs | 36 ++++-------- .../Compressors/ArcLzw/ArcLzwStream.cs | 42 +++++++++----- .../Arc/ArcAllocationTests.cs | 58 +++++++++++++++++++ 6 files changed, 110 insertions(+), 40 deletions(-) create mode 100644 tests/SharpCompress.Test/Arc/ArcAllocationTests.cs diff --git a/docs/USAGE.md b/docs/USAGE.md index 18249a87..d78ae51e 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -475,3 +475,5 @@ await archive.WriteToDirectoryAsync( ## Malformed input and memory use ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size. + +ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size. diff --git a/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs b/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs index 4af92288..134e3f86 100644 --- a/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs +++ b/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs @@ -37,6 +37,12 @@ public partial class ArcFilePart .ConfigureAwait(false); break; case CompressionType.Crunched: + if (Header.CompressedSize > int.MaxValue) + { + throw new InvalidFormatException( + "ARC: compressed size exceeds the supported range." + ); + } if (Header.OriginalSize > 128 * 1024) { throw new NotSupportedException( diff --git a/src/SharpCompress/Common/Arc/ArcFilePart.cs b/src/SharpCompress/Common/Arc/ArcFilePart.cs index d104396e..b37347fa 100644 --- a/src/SharpCompress/Common/Arc/ArcFilePart.cs +++ b/src/SharpCompress/Common/Arc/ArcFilePart.cs @@ -52,6 +52,12 @@ public partial class ArcFilePart : FilePart compressedStream = SqueezeStream.Create(_stream, (int)Header.CompressedSize); break; case CompressionType.Crunched: + if (Header.CompressedSize > int.MaxValue) + { + throw new InvalidFormatException( + "ARC: compressed size exceeds the supported range." + ); + } if (Header.OriginalSize > 128 * 1024) { throw new NotSupportedException( diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs index 56496c9a..3ab2030b 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs @@ -1,8 +1,10 @@ using System; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.ArcLzw; @@ -15,37 +17,19 @@ public partial class ArcLzwStream CancellationToken cancellationToken ) { - if (_processed) + cancellationToken.ThrowIfCancellationRequested(); + if (count == 0) { return 0; } - _processed = true; - var data = new byte[_compressedSize]; - int totalRead = 0; - while (totalRead < _compressedSize) + if (!_processed) { - int read = await _stream - .ReadAsync(data, totalRead, _compressedSize - totalRead, cancellationToken) - .ConfigureAwait(false); - if (read == 0) - { - break; - } - totalRead += read; + using var limited = new ReadOnlySubStream(_stream, _compressedSize); + using var data = new MemoryStream(); + await limited.CopyToAsync(data, 81920, cancellationToken).ConfigureAwait(false); + FinishDecompression(data); } - var decoded = Decompress(data, _useCrunched); - var result = decoded.Count; - if (_useCrunched) - { - var unpacked = RLE.UnpackRLE(decoded.ToArray()); - unpacked.CopyTo(buffer, 0); - result = unpacked.Count; - } - else - { - decoded.CopyTo(buffer, 0); - } - return result; + return Read(buffer, offset, count); } #if !LEGACY_DOTNET diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs index 542d991a..c0f763dd 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.ArcLzw; @@ -13,6 +14,8 @@ public partial class ArcLzwStream : Stream private bool _processed; private bool _useCrunched; private int _compressedSize; + private byte[]? _decoded; + private int _decodedPosition; private const int BITS = 12; private const int CRUNCH_BITS = 12; @@ -34,6 +37,10 @@ public partial class ArcLzwStream : Stream public ArcLzwStream(Stream stream, int compressedSize, bool useCrunched = true) { + if (compressedSize < 0) + { + throw new InvalidFormatException("ARC: invalid compressed size."); + } _stream = stream; _useCrunched = useCrunched; _compressedSize = compressedSize; @@ -177,28 +184,35 @@ public partial class ArcLzwStream : Stream public override int Read(byte[] buffer, int offset, int count) { - if (_processed) + if (count == 0) { return 0; } - _processed = true; - var data = new byte[_compressedSize]; - _stream.Read(data, 0, _compressedSize); - var decoded = Decompress(data, _useCrunched); - var result = decoded.Count; - if (_useCrunched) + if (!_processed) { - var unpacked = RLE.UnpackRLE(decoded.ToArray()); - unpacked.CopyTo(buffer, 0); - result = unpacked.Count; - } - else - { - decoded.CopyTo(buffer, 0); + using var limited = new ReadOnlySubStream(_stream, _compressedSize); + using var data = new MemoryStream(); + limited.CopyTo(data); + FinishDecompression(data); } + var result = Math.Min(count, _decoded!.Length - _decodedPosition); + Buffer.BlockCopy(_decoded, _decodedPosition, buffer, offset, result); + _decodedPosition += result; return result; } + private void FinishDecompression(MemoryStream data) + { + if (data.Length != _compressedSize) + { + throw new IncompleteArchiveException("Truncated ARC compressed data."); + } + // Buffer only bytes actually read, never the size claimed by the ARC header. + var decoded = Decompress(data.ToArray(), _useCrunched); + _decoded = _useCrunched ? RLE.UnpackRLE(decoded.ToArray()).ToArray() : decoded.ToArray(); + _processed = true; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); diff --git a/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs b/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs new file mode 100644 index 00000000..9c16694e --- /dev/null +++ b/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs @@ -0,0 +1,58 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.ArcLzw; +using Xunit; + +namespace SharpCompress.Test.Arc; + +public class ArcAllocationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Arc_Rejects_Huge_Truncated_Compressed_Data(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 12, 65, 0 }); + using var stream = new ArcLzwStream(input, int.MaxValue); + var buffer = new byte[1]; + if (useAsync) + { + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => stream.Read(buffer, 0, 1)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Arc_Preserves_Output_Across_Small_Reads(bool useAsync) + { + // Two nine-bit literals, followed by a byte belonging to the next entry. + using var input = new MemoryStream(new byte[] { 65, 132, 0, 99 }); + using var stream = new ArcLzwStream(input, 3, false); + var buffer = new byte[3]; + foreach (var expected in new byte[] { 65, 66 }) + { + var read = useAsync + ? await stream.ReadAsync(buffer, 1, 1, CancellationToken.None) + : stream.Read(buffer, 1, 1); + Assert.Equal(1, read); + Assert.Equal(expected, buffer[1]); + Assert.Equal(0, buffer[0]); + } + Assert.Equal(0, stream.Read(buffer, 0, 1)); + Assert.Equal(99, input.ReadByte()); + } + + [Fact] + public void Arc_Rejects_Negative_Compressed_Size() => + Assert.Throws(() => new ArcLzwStream(Stream.Null, -1)); +} From 8a9b76dd445d5f98265188ff05ce2f8adb809dea Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 16 Oct 2026 15:35:27 +0100 Subject: [PATCH 4/7] Grow Shrink output from decoded bytes and stop at input EOF --- docs/USAGE.md | 2 + .../Compressors/Shrink/BitStream.cs | 12 ++-- .../Compressors/Shrink/HwUnshrink.cs | 39 ++++++++++++ .../Compressors/Shrink/ShrinkStream.Async.cs | 8 +-- .../Compressors/Shrink/ShrinkStream.cs | 37 ++++++----- .../Zip/ShrinkAllocationTests.cs | 63 +++++++++++++++++++ 6 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs diff --git a/docs/USAGE.md b/docs/USAGE.md index d78ae51e..511ac6b4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -477,3 +477,5 @@ await archive.WriteToDirectoryAsync( ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size. ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size. + +ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected. diff --git a/src/SharpCompress/Compressors/Shrink/BitStream.cs b/src/SharpCompress/Compressors/Shrink/BitStream.cs index 77df3fa3..84841662 100644 --- a/src/SharpCompress/Compressors/Shrink/BitStream.cs +++ b/src/SharpCompress/Compressors/Shrink/BitStream.cs @@ -9,6 +9,7 @@ internal class BitStream private int _bitIdx; private int _bitsLeft; private ulong _bitBuffer; + private bool _lastReadValid; private static uint[] _maskBits = new uint[17] { 0U, @@ -62,6 +63,11 @@ internal class BitStream _bitsLeft += 8; } } + _lastReadValid = _bitsLeft >= nbits; + if (!_lastReadValid) + { + return 0; + } result = (int)(_bitBuffer & _maskBits[nbits]); _bitBuffer >>= nbits; _bitsLeft -= nbits; @@ -70,10 +76,6 @@ internal class BitStream public bool Advance(int count) { - if (_byteIdx > _srcLen) - { - return false; - } - return true; + return _lastReadValid; } } diff --git a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs index 81e411c1..25170093 100644 --- a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs +++ b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs @@ -238,6 +238,43 @@ public class HwUnshrink byte[] dst, int dstCap, out int dstUsed + ) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, false); + + internal static UnshrnkStatus UnshrinkGrowing( + byte[] src, + int dstCap, + out byte[] dst, + out int dstUsed + ) + { + dst = Array.Empty(); + return UnshrinkCore(src, src.Length, out _, ref dst, dstCap, out dstUsed, true); + } + + private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, bool grow) + { + if (!grow) + { + return; + } + // A code string is bounded by the code table's ushort length. Reserve at most + // one string ahead of actual output, not the size advertised by the ZIP header. + var required = (int)Math.Min(dstCap, (long)dstPos + ushort.MaxValue + 1); + if (required > dst.Length) + { + var capacity = (int)Math.Min(dstCap, Math.Max(required, (long)dst.Length * 2)); + Array.Resize(ref dst, capacity); + } + } + + private static UnshrnkStatus UnshrinkCore( + byte[] src, + int srcLen, + out int srcUsed, + ref byte[] dst, + int dstCap, + out int dstUsed, + bool grow ) { var codeTab = new CodeTabEntry[HASHTAB_SIZE]; @@ -279,6 +316,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, grow); firstByte = (byte)currCode; dst[dstPos] = (byte)currCode; codeTab[currCode].lastDstPos = dstPos; @@ -301,6 +339,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, grow); // Handle KwKwK: next code used before being added. if (currCode == queue.codes[queue.nextIdx]) { diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs index c5f8c49a..49895df3 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs @@ -31,12 +31,8 @@ internal partial class ShrinkStream : Stream using var srcMs = new MemoryStream(); await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false); var src = srcMs.ToArray(); - var srcLen = src.Length; - - // Decompress synchronously (CPU-bound operation) - HwUnshrink.Unshrink(src, srcLen, out _, _byteOut, (int)_uncompressedSize, out var dstUsed); - _outBytesCount = dstUsed; - _decompressed = true; + // Decompress synchronously (CPU-bound operation). + Decode(src); } public override async Task ReadAsync( diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs index a3e680b2..a1f34259 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs @@ -9,14 +9,14 @@ internal partial class ShrinkStream : Stream private readonly Stream _inStream; private readonly long _uncompressedSize; - private readonly byte[] _byteOut; + private byte[] _byteOut = Array.Empty(); private long _outBytesCount; private bool _decompressed; private long _position; public ShrinkStream(Stream stream, long uncompressedSize) { - if (uncompressedSize > int.MaxValue) + if (uncompressedSize < 0 || uncompressedSize > int.MaxValue) { throw new InvalidFormatException( $"Shrink: declared uncompressed size {uncompressedSize} exceeds maximum supported size." @@ -26,7 +26,6 @@ internal partial class ShrinkStream : Stream _inStream = stream; _uncompressedSize = uncompressedSize; - _byteOut = new byte[(int)_uncompressedSize]; _outBytesCount = 0L; } @@ -56,19 +55,7 @@ internal partial class ShrinkStream : Stream using var srcMs = new MemoryStream(); _inStream.CopyTo(srcMs); var src = srcMs.ToArray(); - var srcLen = src.Length; - - HwUnshrink.Unshrink( - src, - srcLen, - out _, - _byteOut, - (int)_uncompressedSize, - out var dstUsed - ); - _outBytesCount = dstUsed; - _decompressed = true; - _position = 0; + Decode(src); } long remaining = _outBytesCount - _position; @@ -86,6 +73,24 @@ internal partial class ShrinkStream : Stream public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + private void Decode(byte[] src) + { + var status = HwUnshrink.UnshrinkGrowing( + src, + (int)_uncompressedSize, + out var output, + out var used + ); + if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize) + { + throw new InvalidFormatException("Shrink: invalid or truncated compressed data."); + } + _byteOut = output; + _outBytesCount = used; + _decompressed = true; + _position = 0; + } + public override void SetLength(long value) => throw new NotImplementedException(); public override void Write(byte[] buffer, int offset, int count) => diff --git a/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs new file mode 100644 index 00000000..31e0e317 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs @@ -0,0 +1,63 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.Shrink; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class ShrinkAllocationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Shrink_Rejects_Huge_Truncated_Output(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 65, 0 }); + if (useAsync) + { + await Assert.ThrowsAsync(async () => + { + using var stream = await ShrinkStream.CreateAsync( + input, + int.MaxValue, + CancellationToken.None + ); + }); + } + else + { + using var stream = new ShrinkStream(input, int.MaxValue); + Assert.Throws(() => stream.ReadByte()); + } + } + + [Fact] + public void Shrink_Grows_Only_For_Actual_Output() + { + var status = HwUnshrink.UnshrinkGrowing( + new byte[] { 65, 132, 0 }, + int.MaxValue, + out var output, + out var used + ); + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(2, used); + Assert.Equal(65, output[0]); + Assert.Equal(66, output[1]); + Assert.InRange(output.Length, 2, 131072); + } + + [Fact] + public void Shrink_Preserves_Fixed_Buffer_API() + { + var output = new byte[2]; + Assert.Equal( + HwUnshrink.UnshrnkStatus.Ok, + HwUnshrink.Unshrink(new byte[] { 65, 132, 0 }, 3, out _, output, 2, out var used) + ); + Assert.Equal(2, used); + Assert.Equal(new byte[] { 65, 66 }, output); + } +} From 2369f10b63c28c258534b05415259c57e2cddf45 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 16 Oct 2026 15:39:56 +0100 Subject: [PATCH 5/7] Grow LZMA dictionary storage only as decoded history requires --- docs/USAGE.md | 2 + .../Compressors/LZMA/LZ/LzOutWindow.Async.cs | 7 +- .../Compressors/LZMA/LZ/LzOutWindow.cs | 34 ++- .../Compressors/LZMA/LzmaStream.cs | 5 + .../Streams/LzmaDictionaryAllocationTests.cs | 226 ++++++++++++++++++ 5 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs diff --git a/docs/USAGE.md b/docs/USAGE.md index 511ac6b4..f17bd9a6 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -479,3 +479,5 @@ ZIP64 end-record extensible data is skipped with bounded buffering. Invalid reco ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size. ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected. + +LZMA/LZMA2 dictionary storage starts small and grows with decoded history, up to the supported dictionary size. These safeguards avoid allocations based solely on forged sizes; they do not impose an overall decompressed-output or process-memory limit. diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs index 011fdf2e..22d51c43 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs @@ -88,6 +88,7 @@ internal partial class OutWindow : IAsyncDisposable var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize); if (fastSize >= 2) { + EnsureCapacity(_pos + fastSize); _buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize)); _pos += fastSize; pos += fastSize; @@ -113,6 +114,7 @@ internal partial class OutWindow : IAsyncDisposable public async ValueTask PutByteAsync(byte b, CancellationToken cancellationToken = default) { + EnsureCapacity(_pos + 1); _buffer[_pos++] = b; _total++; if (_pos >= _windowSize) @@ -132,7 +134,7 @@ internal partial class OutWindow : IAsyncDisposable { cancellationToken.ThrowIfCancellationRequested(); - var curSize = _windowSize - _pos; + var curSize = Math.Min(4096, _windowSize - _pos); if (curSize > _limit - _total) { curSize = (int)(_limit - _total); @@ -141,6 +143,7 @@ internal partial class OutWindow : IAsyncDisposable { curSize = size; } + EnsureCapacity(_pos + curSize); var numReadBytes = await stream .ReadAsync(_buffer, _pos, curSize, cancellationToken) .ConfigureAwait(false); @@ -166,7 +169,7 @@ internal partial class OutWindow : IAsyncDisposable stream.Position = len - size; _total = 0; _limit = size; - _pos = _windowSize - size; + _pos = 0; await CopyStreamAsync(stream, size).ConfigureAwait(false); if (_pos == _windowSize) { diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs index 78300446..49a53bc2 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs @@ -24,21 +24,23 @@ internal partial class OutWindow : IDisposable public long Total => _total; + internal int BufferCapacity => _buffer?.Length ?? 0; + public void Create(int windowSize) { if (windowSize <= 0) { throw new InvalidFormatException($"LZMA: invalid dictionary size {windowSize}"); } - if (_windowSize != windowSize) + if (_windowSize != windowSize || _buffer is null) { + var replacement = ArrayPool.Shared.Rent(Math.Min(windowSize, 4096)); if (_buffer is not null) { ArrayPool.Shared.Return(_buffer); } - _buffer = ArrayPool.Shared.Rent(windowSize); + _buffer = replacement; } - _buffer[windowSize - 1] = 0; _windowSize = windowSize; _pos = 0; _streamPos = 0; @@ -47,6 +49,20 @@ internal partial class OutWindow : IDisposable _limit = 0; } + private void EnsureCapacity(int required) + { + if (required <= _buffer.Length) + { + return; + } + // Grow from emitted history, never from the advertised dictionary or output size. + var capacity = (int)Math.Min(_windowSize, Math.Max(required, (long)_buffer.Length * 2)); + var replacement = ArrayPool.Shared.Rent(capacity); + _buffer.AsSpan(0, _pos).CopyTo(replacement); + ArrayPool.Shared.Return(_buffer); + _buffer = replacement; + } + public void Dispose() { ReleaseStream(); @@ -77,7 +93,7 @@ internal partial class OutWindow : IDisposable stream.Position = len - size; _total = 0; _limit = size; - _pos = _windowSize - size; + _pos = 0; CopyStream(stream, size); if (_pos == _windowSize) { @@ -141,6 +157,7 @@ internal partial class OutWindow : IDisposable var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize); if (fastSize >= 2) { + EnsureCapacity(_pos + fastSize); _buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize)); _pos += fastSize; pos += fastSize; @@ -166,6 +183,7 @@ internal partial class OutWindow : IDisposable public void PutByte(byte b) { + EnsureCapacity(_pos + 1); _buffer[_pos++] = b; _total++; if (_pos >= _windowSize) @@ -176,6 +194,11 @@ internal partial class OutWindow : IDisposable public byte GetByte(int distance) { + // The initial literal context is zero; no dictionary history has been emitted yet. + if (_total == 0 && distance == 0) + { + return 0; + } var pos = _pos - distance - 1; if (pos < 0) { @@ -189,7 +212,7 @@ internal partial class OutWindow : IDisposable var size = len; while (size > 0 && _pos < _windowSize && _total < _limit) { - var curSize = _windowSize - _pos; + var curSize = Math.Min(4096, _windowSize - _pos); if (curSize > _limit - _total) { curSize = (int)(_limit - _total); @@ -198,6 +221,7 @@ internal partial class OutWindow : IDisposable { curSize = size; } + EnsureCapacity(_pos + curSize); var numReadBytes = stream.Read(_buffer, _pos, curSize); if (numReadBytes == 0) { diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index e4454771..744e0a29 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -65,6 +65,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } else { + // Larger standard dictionaries exceed this decoder's positive Int32 window range. + if (properties[0] > 37) + { + throw new InvalidParamException(); + } _dictionarySize = 2 | (properties[0] & 1); _dictionarySize <<= (properties[0] >> 1) + 11; diff --git a/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs b/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs new file mode 100644 index 00000000..6e8d536f --- /dev/null +++ b/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs @@ -0,0 +1,226 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.LZMA.LZ; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class LzmaDictionaryAllocationTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Lzma_Raw_Huge_Dictionary_Decodes_With_Stream_And_Direct_APIs( + bool useAsync, + bool direct + ) + { + var expected = new byte[20000]; + new Random(123).NextBytes(expected); + using var packed = new MemoryStream(); + byte[] properties; + using ( + var encoder = LzmaStream.Create(new LzmaEncoderProperties(false, 4096), false, packed) + ) + { + properties = (byte[])encoder.Properties.Clone(); + encoder.Write(expected, 0, expected.Length); + } + // The compressed payload needs only a small dictionary, regardless of its advertised size. + properties[1] = 255; + properties[2] = 255; + properties[3] = 255; + properties[4] = 127; + using var input = new MemoryStream(packed.ToArray()); + using var output = new MemoryStream(); + if (direct) + { + using var decoder = new Decoder(); + decoder.SetDecoderProperties(properties); + if (useAsync) + { + await decoder.CodeAsync( + input, + output, + input.Length, + expected.Length, + null!, + CancellationToken.None + ); + } + else + { + decoder.Code(input, output, input.Length, expected.Length, null!); + } + } + else if (useAsync) + { + using var decoder = await LzmaStream.CreateAsync( + properties, + input, + input.Length, + expected.Length + ); + await decoder.CopyToAsync(output, 137, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create(properties, input, input.Length, expected.Length); + decoder.CopyTo(output, 137); + } + Assert.Equal(expected, output.ToArray()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_Truncated_Uncompressed_Chunk_Does_Not_Allocate_From_Read_Size( + bool useAsync + ) + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + window.SetLimit(int.MaxValue); + using var input = new MemoryStream(new byte[] { 1 }); + if (useAsync) + { + await Assert.ThrowsAnyAsync(async () => + await window.CopyStreamAsync(input, int.MaxValue, CancellationToken.None) + ); + } + else + { + Assert.ThrowsAny(() => window.CopyStream(input, int.MaxValue)); + } + Assert.InRange(window.BufferCapacity, 1, 8192); + } + + [Theory] + [InlineData(38)] + [InlineData(40)] + [InlineData(64)] + [InlineData(255)] + public void Lzma2_Rejects_Unsupported_Dictionary_Properties(int property) + { + Assert.ThrowsAny(() => + LzmaStream.Create(new byte[] { (byte)property }, Stream.Null) + ); + } + + [Fact] + public void Lzma_Huge_Dictionary_Starts_Small_And_Grows_With_History() + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + Assert.InRange(window.BufferCapacity, 1, 4096); + Assert.Equal(0, window.GetByte(0)); + window.SetLimit(20000); + for (var i = 0; i < 20000; i++) + { + window.PutByte((byte)i); + } + Assert.InRange(window.BufferCapacity, 20000, 32768); + for (var i = 0; i < 20000; i++) + { + Assert.Equal((byte)i, window.GetByte(19999 - i)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_History_Survives_Growth_Wrap_And_Pending_Matches(bool useAsync) + { + using var window = new OutWindow(); + window.Create(8192); + var output = new byte[8192]; + window.SetLimit(8192); + for (var i = 0; i < output.Length; i++) + { + if (useAsync) + { + await window.PutByteAsync((byte)i, CancellationToken.None); + } + else + { + window.PutByte((byte)i); + } + } + Assert.Equal(output.Length, window.Read(output, 0, output.Length)); + window.SetLimit(10); + if (useAsync) + { + await window.CopyBlockAsync(8191, 20, CancellationToken.None); + } + else + { + window.CopyBlock(8191, 20); + } + Assert.True(window.HasPending); + window.SetLimit(10); + if (useAsync) + { + await window.CopyPendingAsync(CancellationToken.None); + } + else + { + window.CopyPending(); + } + Assert.False(window.HasPending); + Assert.Equal(20, window.Read(output, 0, output.Length)); + for (var i = 0; i < 20; i++) + { + Assert.Equal((byte)i, output[i]); + } + window.Reset(); + Assert.Equal(0, window.Total); + Assert.Equal(0, window.GetByte(0)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_Preset_Dictionary_Does_Not_Allocate_Declared_Window(bool useAsync) + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + using var preset = new MemoryStream(new byte[] { 1, 2, 3 }); + if (useAsync) + { + await window.TrainAsync(preset); + } + else + { + window.Train(preset); + } + Assert.Equal(3, window.GetByte(0)); + Assert.Equal(1, window.GetByte(2)); + Assert.InRange(window.BufferCapacity, 3, 4096); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma2_Tiny_Stream_With_Huge_Dictionary_Decodes(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 }); + using var output = new MemoryStream(); + if (useAsync) + { + using var decoder = await LzmaStream.CreateAsync(new byte[] { 37 }, input, -1, -1); + await decoder.CopyToAsync(output, 81920, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create(new byte[] { 37 }, input); + decoder.CopyTo(output); + } + Assert.Equal(new byte[] { 65 }, output.ToArray()); + } +} From 7fc5c0e56577c912024987fc39f2bb853fc149a1 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 16 Oct 2026 15:44:44 +0100 Subject: [PATCH 6/7] Use pooled memory streams for temporary production buffers --- .../Compressors/ArcLzw/ArcLzwStream.Async.cs | 2 +- src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs | 4 ++-- src/SharpCompress/Compressors/PPMd/PpmdStream.cs | 3 ++- .../Compressors/Shrink/ShrinkStream.Async.cs | 3 ++- src/SharpCompress/Compressors/Shrink/ShrinkStream.cs | 3 ++- .../Writers/SevenZip/SevenZipWriter.Async.cs | 8 ++++---- src/SharpCompress/Writers/Zip/ZipWriter.Async.cs | 7 ++++--- 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs index 3ab2030b..96d6da7d 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs @@ -25,7 +25,7 @@ public partial class ArcLzwStream if (!_processed) { using var limited = new ReadOnlySubStream(_stream, _compressedSize); - using var data = new MemoryStream(); + using var data = new PooledMemoryStream(); await limited.CopyToAsync(data, 81920, cancellationToken).ConfigureAwait(false); FinishDecompression(data); } diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs index c0f763dd..10a32111 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs @@ -191,7 +191,7 @@ public partial class ArcLzwStream : Stream if (!_processed) { using var limited = new ReadOnlySubStream(_stream, _compressedSize); - using var data = new MemoryStream(); + using var data = new PooledMemoryStream(); limited.CopyTo(data); FinishDecompression(data); } @@ -201,7 +201,7 @@ public partial class ArcLzwStream : Stream return result; } - private void FinishDecompression(MemoryStream data) + private void FinishDecompression(PooledMemoryStream data) { if (data.Length != _compressedSize) { diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index b48cf4cf..ad958fa4 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -202,7 +202,8 @@ public class PpmdStream : Stream, IAsyncDisposable _isDisposed = true; if (_compress) { - await _model.EncodeBlockAsync(_stream, new MemoryStream(), true).ConfigureAwait(false); + using var input = new PooledMemoryStream(); + await _model.EncodeBlockAsync(_stream, input, true).ConfigureAwait(false); } _modelH?.Dispose(); _modelH = null; diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs index 49895df3..5d6f7e61 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Compressors.Shrink; @@ -28,7 +29,7 @@ internal partial class ShrinkStream : Stream // Read actual compressed data from the stream rather than pre-allocating based on the // declared compressed size, which may be crafted to cause an OutOfMemoryException. // The stream is already bounded by ReadOnlySubStream in ZipFilePart. - using var srcMs = new MemoryStream(); + using var srcMs = new PooledMemoryStream(); await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false); var src = srcMs.ToArray(); // Decompress synchronously (CPU-bound operation). diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs index a1f34259..51ee203d 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs @@ -1,6 +1,7 @@ using System; using System.IO; using SharpCompress.Common; +using SharpCompress.IO; namespace SharpCompress.Compressors.Shrink; @@ -52,7 +53,7 @@ internal partial class ShrinkStream : Stream // Read actual compressed data from the stream rather than pre-allocating based on the // declared compressed size, which may be crafted to cause an OutOfMemoryException. // The stream is already bounded by ReadOnlySubStream in ZipFilePart. - using var srcMs = new MemoryStream(); + using var srcMs = new PooledMemoryStream(); _inStream.CopyTo(srcMs); var src = srcMs.ToArray(); Decode(src); diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs index 9b24371d..ce5b9021 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs @@ -53,7 +53,7 @@ public partial class SevenZipWriter var filesInfo = new SevenZipFilesInfoWriter { Entries = entries.ToArray() }; // Write header to a temporary stream first - using var headerStream = new MemoryStream(); + using var headerStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteRawHeader(headerStream, mainStreamsInfo, filesInfo); // Optionally compress the header @@ -68,7 +68,7 @@ public partial class SevenZipWriter } private async ValueTask WriteCompressedHeaderAsync( - MemoryStream rawHeaderStream, + PooledMemoryStream rawHeaderStream, long endOfPackedData ) { @@ -95,7 +95,7 @@ public partial class SevenZipWriter }; // Write encoded header to a second temporary stream - using var encodedHeaderStream = new MemoryStream(); + using var encodedHeaderStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteEncodedHeader(encodedHeaderStream, headerStreamsInfo); // Write the encoded header to the output @@ -123,7 +123,7 @@ public partial class SevenZipWriter } private async ValueTask WriteRawHeaderToOutputAsync( - MemoryStream rawHeaderStream, + PooledMemoryStream rawHeaderStream, long endOfPackedData ) { diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs index b934c6e1..fbf5ab07 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; +using SharpCompress.IO; namespace SharpCompress.Writers.Zip; @@ -24,7 +25,7 @@ public partial class ZipWriter // Buffer the entire central directory + end record into memory, then write async. // This avoids synchronous writes to the underlying stream during finalization. - using var ms = new MemoryStream(); + using var ms = new PooledMemoryStream(); ulong size = 0; foreach (var entry in entries) { @@ -149,9 +150,9 @@ public partial class ZipWriter CancellationToken cancellationToken ) { - // Build the header synchronously into a MemoryStream, then async-copy to OutputStream. + // Build the header synchronously into a pooled stream, then async-copy to OutputStream. // This avoids any synchronous writes to the potentially async-only output stream. - using var ms = new MemoryStream(); + using var ms = new PooledMemoryStream(); var outputStream = OutputStream.NotNull(); var result = WriteHeader( ms, From 7642f76f388e385b450a2b5b0e87929d0a04285f Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 19 Oct 2026 10:10:09 +0100 Subject: [PATCH 7/7] changes fixes to use pooled arrays --- .../Headers/Zip64DirectoryEndHeader.Async.cs | 21 +- .../Zip/Headers/Zip64DirectoryEndHeader.cs | 31 ++- .../Compressors/Shrink/HwUnshrink.cs | 37 +++- .../Compressors/Shrink/ShrinkStream.Async.cs | 5 +- .../Compressors/Shrink/ShrinkStream.cs | 85 ++++++-- .../Streams/BufferPoolLifetimeTests.cs | 186 +++++++++++++++++- .../Zip/ShrinkAllocationTests.cs | 21 +- 7 files changed, 341 insertions(+), 45 deletions(-) diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs index df9e7cbe..bc4b5409 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs @@ -23,12 +23,23 @@ internal partial class Zip64DirectoryEndHeader await reader.ReadUInt64Async().ConfigureAwait(false); var remaining = SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; - var buffer = new byte[4096]; - while (remaining > 0) + if (remaining == 0) { - var size = (int)Math.Min(buffer.Length, remaining); - await reader.ReadBytesAsync(buffer, 0, size).ConfigureAwait(false); - remaining -= size; + return; + } + var buffer = _pool.Rent(4096); + try + { + while (remaining > 0) + { + var size = (int)Math.Min(4096, remaining); + await reader.ReadBytesAsync(buffer, 0, size).ConfigureAwait(false); + remaining -= size; + } + } + finally + { + _pool.Return(buffer); } } } diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs index 96d01637..541f2ceb 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs @@ -1,12 +1,18 @@ using System; +using System.Buffers; using System.IO; namespace SharpCompress.Common.Zip.Headers; internal partial class Zip64DirectoryEndHeader : ZipHeader { + private readonly ArrayPool _pool; + public Zip64DirectoryEndHeader() - : base(ZipHeaderType.Zip64DirectoryEnd) { } + : this(ArrayPool.Shared) { } + + internal Zip64DirectoryEndHeader(ArrayPool pool) + : base(ZipHeaderType.Zip64DirectoryEnd) => _pool = pool; internal override void Read(BinaryReader reader) { @@ -22,15 +28,26 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader // Extensible data is not interpreted. Consume it without allocating from its declared size. var remaining = SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; - var buffer = new byte[4096]; - while (remaining > 0) + if (remaining == 0) { - var read = reader.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); - if (read == 0) + return; + } + var buffer = _pool.Rent(4096); + try + { + while (remaining > 0) { - throw new IncompleteArchiveException("Truncated ZIP64 extensible data."); + var read = reader.Read(buffer, 0, (int)Math.Min(4096, remaining)); + if (read == 0) + { + throw new IncompleteArchiveException("Truncated ZIP64 extensible data."); + } + remaining -= read; } - remaining -= read; + } + finally + { + _pool.Return(buffer); } } diff --git a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs index 25170093..706674bc 100644 --- a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs +++ b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; namespace SharpCompress.Compressors.Shrink; @@ -238,22 +239,36 @@ public class HwUnshrink byte[] dst, int dstCap, out int dstUsed - ) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, false); + ) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, null); internal static UnshrnkStatus UnshrinkGrowing( byte[] src, + int srcLen, int dstCap, + ArrayPool pool, out byte[] dst, out int dstUsed ) { dst = Array.Empty(); - return UnshrinkCore(src, src.Length, out _, ref dst, dstCap, out dstUsed, true); + try + { + return UnshrinkCore(src, srcLen, out _, ref dst, dstCap, out dstUsed, pool); + } + catch + { + if (dst.Length > 0) + { + pool.Return(dst); + dst = Array.Empty(); + } + throw; + } } - private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, bool grow) + private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, ArrayPool? pool) { - if (!grow) + if (pool is null) { return; } @@ -263,7 +278,13 @@ public class HwUnshrink if (required > dst.Length) { var capacity = (int)Math.Min(dstCap, Math.Max(required, (long)dst.Length * 2)); - Array.Resize(ref dst, capacity); + var replacement = pool.Rent(capacity); + dst.AsSpan(0, dstPos).CopyTo(replacement); + if (dst.Length > 0) + { + pool.Return(dst); + } + dst = replacement; } } @@ -274,7 +295,7 @@ public class HwUnshrink ref byte[] dst, int dstCap, out int dstUsed, - bool grow + ArrayPool? pool ) { var codeTab = new CodeTabEntry[HASHTAB_SIZE]; @@ -316,7 +337,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } - ReserveOutput(ref dst, dstPos, dstCap, grow); + ReserveOutput(ref dst, dstPos, dstCap, pool); firstByte = (byte)currCode; dst[dstPos] = (byte)currCode; codeTab[currCode].lastDstPos = dstPos; @@ -339,7 +360,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } - ReserveOutput(ref dst, dstPos, dstCap, grow); + ReserveOutput(ref dst, dstPos, dstCap, pool); // Handle KwKwK: next code used before being added. if (currCode == queue.codes[queue.nextIdx]) { diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs index 5d6f7e61..5185c401 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs @@ -31,9 +31,8 @@ internal partial class ShrinkStream : Stream // The stream is already bounded by ReadOnlySubStream in ZipFilePart. using var srcMs = new PooledMemoryStream(); await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false); - var src = srcMs.ToArray(); // Decompress synchronously (CPU-bound operation). - Decode(src); + Decode(srcMs); } public override async Task ReadAsync( @@ -43,6 +42,7 @@ internal partial class ShrinkStream : Stream CancellationToken cancellationToken ) { + ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); if (!_decompressed) @@ -69,6 +69,7 @@ internal partial class ShrinkStream : Stream CancellationToken cancellationToken = default ) { + ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); if (!_decompressed) diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs index 51ee203d..1a6d69e5 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.IO; using SharpCompress.Common; using SharpCompress.IO; @@ -8,14 +9,19 @@ namespace SharpCompress.Compressors.Shrink; internal partial class ShrinkStream : Stream { private readonly Stream _inStream; + private readonly ArrayPool _pool; private readonly long _uncompressedSize; private byte[] _byteOut = Array.Empty(); private long _outBytesCount; private bool _decompressed; + private bool _disposed; private long _position; public ShrinkStream(Stream stream, long uncompressedSize) + : this(stream, uncompressedSize, ArrayPool.Shared) { } + + internal ShrinkStream(Stream stream, long uncompressedSize, ArrayPool pool) { if (uncompressedSize < 0 || uncompressedSize > int.MaxValue) { @@ -25,12 +31,13 @@ internal partial class ShrinkStream : Stream } _inStream = stream; + _pool = pool; _uncompressedSize = uncompressedSize; _outBytesCount = 0L; } - public override bool CanRead => true; + public override bool CanRead => !_disposed; public override bool CanSeek => true; @@ -48,6 +55,7 @@ internal partial class ShrinkStream : Stream public override int Read(byte[] buffer, int offset, int count) { + ThrowIfDisposed(); if (!_decompressed) { // Read actual compressed data from the stream rather than pre-allocating based on the @@ -55,8 +63,7 @@ internal partial class ShrinkStream : Stream // The stream is already bounded by ReadOnlySubStream in ZipFilePart. using var srcMs = new PooledMemoryStream(); _inStream.CopyTo(srcMs); - var src = srcMs.ToArray(); - Decode(src); + Decode(srcMs); } long remaining = _outBytesCount - _position; @@ -74,22 +81,68 @@ internal partial class ShrinkStream : Stream public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); - private void Decode(byte[] src) + private void Decode(PooledMemoryStream source) { - var status = HwUnshrink.UnshrinkGrowing( - src, - (int)_uncompressedSize, - out var output, - out var used - ); - if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize) + var length = checked((int)source.Length); + var src = _pool.Rent(length); + var output = Array.Empty(); + try { - throw new InvalidFormatException("Shrink: invalid or truncated compressed data."); + source.Position = 0; + if (source.Read(src, 0, length) != length) + { + throw new InvalidFormatException("Shrink: truncated compressed data."); + } + // A pool rental may be larger than the compressed data; never decode its unused tail. + var status = HwUnshrink.UnshrinkGrowing( + src, + length, + (int)_uncompressedSize, + _pool, + out output, + out var used + ); + if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize) + { + throw new InvalidFormatException("Shrink: invalid or truncated compressed data."); + } + _byteOut = output; + output = Array.Empty(); + _outBytesCount = used; + _decompressed = true; + _position = 0; + } + finally + { + _pool.Return(src); + if (output.Length > 0) + { + _pool.Return(output); + } + } + } + + protected override void Dispose(bool disposing) + { + if (!_disposed) + { + _disposed = true; + var output = _byteOut; + _byteOut = Array.Empty(); + if (output.Length > 0) + { + _pool.Return(output); + } + } + base.Dispose(disposing); + } + + private void ThrowIfDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ShrinkStream)); } - _byteOut = output; - _outBytesCount = used; - _decompressed = true; - _position = 0; } public override void SetLength(long value) => throw new NotImplementedException(); diff --git a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs index a2c030e3..874cca29 100644 --- a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs +++ b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs @@ -6,9 +6,11 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Tar.Headers; +using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.LZMA.LZ; using SharpCompress.Compressors.LZMA.Utilities; +using SharpCompress.Compressors.Shrink; using SharpCompress.IO; using Xunit; @@ -16,6 +18,184 @@ namespace SharpCompress.Test.Streams; public class BufferPoolLifetimeTests { + [Theory] + [InlineData(false, 0, false)] + [InlineData(true, 0, false)] + [InlineData(false, 10000, false)] + [InlineData(true, 10000, false)] + [InlineData(false, 1, true)] + [InlineData(true, 1, true)] + public async Task Zip64_ExtensibleData_ReturnsBuffer( + bool useAsync, + int payloadSize, + bool truncated + ) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(); + using (var writer = new BinaryWriter(input, System.Text.Encoding.UTF8, true)) + { + writer.Write((ulong)(44 + payloadSize + (truncated ? 10000 : 0))); + writer.Write(new byte[44 + payloadSize]); + if (!truncated) + { + writer.Write((byte)99); + } + } + input.Position = 0; + var header = new Zip64DirectoryEndHeader(pool); + if (useAsync) + { + using var reader = new AsyncBinaryReader(input, leaveOpen: true); + if (truncated) + { + await Assert.ThrowsAnyAsync(async () => + await header.Read(reader) + ); + } + else + { + await header.Read(reader); + } + } + else + { + using var reader = new BinaryReader(input, System.Text.Encoding.UTF8, true); + if (truncated) + { + Assert.ThrowsAny(() => header.Read(reader)); + } + else + { + header.Read(reader); + } + } + if (!truncated) + { + Assert.Equal(99, input.ReadByte()); + } + Assert.Equal(payloadSize == 0 ? 0 : 1, pool.RentCount); + Assert.Equal(pool.RentCount, pool.ReturnCount); + Assert.Empty(pool.Active); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shrink_Dispose_ReturnsBuffers(bool useAsync, bool asyncDispose) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(new byte[] { 65, 132, 0 }); + using var stream = new ShrinkStream(input, 2, pool); + var buffer = new byte[1]; + foreach (var expected in new byte[] { 65, 66 }) + { + var read = useAsync + ? await stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + : stream.Read(buffer, 0, 1); + Assert.Equal(1, read); + Assert.Equal(expected, buffer[0]); + } + Assert.Equal(0, stream.Read(buffer, 0, 1)); + Assert.Equal(2, pool.RentCount); + Assert.Equal(1, pool.ReturnCount); + Assert.Single(pool.Active); + +#if !LEGACY_DOTNET + if (asyncDispose) + { + await stream.DisposeAsync(); + } + else +#endif + { + stream.Dispose(); + } + stream.Dispose(); + Assert.Equal(2, pool.ReturnCount); + Assert.Empty(pool.Active); + Assert.True(input.CanRead); + Assert.False(stream.CanRead); + Assert.Throws(() => stream.Read(buffer, 0, 1)); + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Shrink_InvalidData_ReturnsBuffers(bool useAsync) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(new byte[] { 65, 132, 0 }); + using var stream = new ShrinkStream(input, int.MaxValue, pool); + var buffer = new byte[1]; + if (useAsync) + { + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => stream.Read(buffer, 0, 1)); + } + Assert.Equal(2, pool.RentCount); + Assert.Equal(2, pool.ReturnCount); + Assert.Empty(pool.Active); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Shrink_Growth_ReturnsBuffers(bool failGrowth) + { + // Literal-only nine-bit codes fill the first output rental without increasing code size. + const int count = 70000; + var input = new byte[(count * 9 + 7) / 8]; + for (var i = 0; i < count; i++) + { + var bit = i * 9; + input[bit / 8] |= (byte)(65 << (bit % 8)); + input[bit / 8 + 1] |= (byte)(65 >> (8 - bit % 8)); + } + var pool = new TrackingPool { FailRentNumber = failGrowth ? 2 : 0 }; + if (failGrowth) + { + Assert.Throws(() => + HwUnshrink.UnshrinkGrowing(input, input.Length, count, pool, out _, out _) + ); + Assert.Equal(1, pool.ReturnCount); + } + else + { + var status = HwUnshrink.UnshrinkGrowing( + input, + input.Length, + count, + pool, + out var output, + out var used + ); + try + { + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(count, used); + Assert.All(output, value => Assert.Equal(65, value)); + Assert.Equal(1, pool.ReturnCount); + } + finally + { + pool.Return(output); + } + Assert.Equal(2, pool.ReturnCount); + } + Assert.Empty(pool.Active); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -370,18 +550,20 @@ public class BufferPoolLifetimeTests internal int RentCount { get; private set; } internal int ReturnCount { get; private set; } internal bool FailNextRent { get; set; } + internal int FailRentNumber { get; set; } + internal int ExtraCapacity { get; set; } public override T[] Rent(int minimumLength) { RentCount++; - if (FailNextRent) + if (FailNextRent || RentCount == FailRentNumber) { FailNextRent = false; #pragma warning disable CA2201 // Simulate allocation failure without exhausting process memory. throw new OutOfMemoryException("Injected pool allocation failure."); #pragma warning restore CA2201 } - var buffer = new T[minimumLength]; + var buffer = new T[minimumLength + ExtraCapacity]; Assert.True(Active.Add(buffer)); return buffer; } diff --git a/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs index 31e0e317..4fd281de 100644 --- a/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs +++ b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -36,17 +37,27 @@ public class ShrinkAllocationTests [Fact] public void Shrink_Grows_Only_For_Actual_Output() { + var pool = ArrayPool.Shared; var status = HwUnshrink.UnshrinkGrowing( new byte[] { 65, 132, 0 }, + 3, int.MaxValue, + pool, out var output, out var used ); - Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); - Assert.Equal(2, used); - Assert.Equal(65, output[0]); - Assert.Equal(66, output[1]); - Assert.InRange(output.Length, 2, 131072); + try + { + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(2, used); + Assert.Equal(65, output[0]); + Assert.Equal(66, output[1]); + Assert.InRange(output.Length, 2, 131072); + } + finally + { + pool.Return(output); + } } [Fact]