diff --git a/docs/USAGE.md b/docs/USAGE.md index 9666d43f..f17bd9a6 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -472,3 +472,12 @@ await archive.WriteToDirectoryAsync( - Support for cancellation via CancellationToken - Better resource utilization in async/await contexts - Compatible with modern .NET async patterns +## Malformed input and memory use + +ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size. + +ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size. + +ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected. + +LZMA/LZMA2 dictionary storage starts small and grows with decoded history, up to the supported dictionary size. These safeguards avoid allocations based solely on forged sizes; they do not impose an overall decompressed-output or process-memory limit. diff --git a/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs b/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs index 4af92288..134e3f86 100644 --- a/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs +++ b/src/SharpCompress/Common/Arc/ArcFilePart.Async.cs @@ -37,6 +37,12 @@ public partial class ArcFilePart .ConfigureAwait(false); break; case CompressionType.Crunched: + if (Header.CompressedSize > int.MaxValue) + { + throw new InvalidFormatException( + "ARC: compressed size exceeds the supported range." + ); + } if (Header.OriginalSize > 128 * 1024) { throw new NotSupportedException( diff --git a/src/SharpCompress/Common/Arc/ArcFilePart.cs b/src/SharpCompress/Common/Arc/ArcFilePart.cs index d104396e..b37347fa 100644 --- a/src/SharpCompress/Common/Arc/ArcFilePart.cs +++ b/src/SharpCompress/Common/Arc/ArcFilePart.cs @@ -52,6 +52,12 @@ public partial class ArcFilePart : FilePart compressedStream = SqueezeStream.Create(_stream, (int)Header.CompressedSize); break; case CompressionType.Crunched: + if (Header.CompressedSize > int.MaxValue) + { + throw new InvalidFormatException( + "ARC: compressed size exceeds the supported range." + ); + } if (Header.OriginalSize > 128 * 1024) { throw new NotSupportedException( diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs index 9e510688..bc4b5409 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Threading.Tasks; using SharpCompress.IO; @@ -8,7 +9,9 @@ internal partial class Zip64DirectoryEndHeader { internal override async ValueTask Read(AsyncBinaryReader reader) { - SizeOfDirectoryEndRecord = (long)await reader.ReadUInt64Async().ConfigureAwait(false); + SizeOfDirectoryEndRecord = ValidateRecordSize( + await reader.ReadUInt64Async().ConfigureAwait(false) + ); VersionMadeBy = await reader.ReadUInt16Async().ConfigureAwait(false); VersionNeededToExtract = await reader.ReadUInt16Async().ConfigureAwait(false); VolumeNumber = await reader.ReadUInt32Async().ConfigureAwait(false); @@ -18,10 +21,25 @@ internal partial class Zip64DirectoryEndHeader DirectorySize = (long)await reader.ReadUInt64Async().ConfigureAwait(false); DirectoryStartOffsetRelativeToDisk = (long) await reader.ReadUInt64Async().ConfigureAwait(false); - var size = (int)( - SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS - ); - DataSector = new byte[size]; - await reader.ReadBytesAsync(DataSector, 0, size).ConfigureAwait(false); + var remaining = + SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; + if (remaining == 0) + { + return; + } + var buffer = _pool.Rent(4096); + try + { + while (remaining > 0) + { + var size = (int)Math.Min(4096, remaining); + await reader.ReadBytesAsync(buffer, 0, size).ConfigureAwait(false); + remaining -= size; + } + } + finally + { + _pool.Return(buffer); + } } } diff --git a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs index 3933b2e0..541f2ceb 100644 --- a/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs @@ -1,15 +1,22 @@ +using System; +using System.Buffers; using System.IO; namespace SharpCompress.Common.Zip.Headers; internal partial class Zip64DirectoryEndHeader : ZipHeader { + private readonly ArrayPool _pool; + public Zip64DirectoryEndHeader() - : base(ZipHeaderType.Zip64DirectoryEnd) { } + : this(ArrayPool.Shared) { } + + internal Zip64DirectoryEndHeader(ArrayPool pool) + : base(ZipHeaderType.Zip64DirectoryEnd) => _pool = pool; internal override void Read(BinaryReader reader) { - SizeOfDirectoryEndRecord = (long)reader.ReadUInt64(); + SizeOfDirectoryEndRecord = ValidateRecordSize(reader.ReadUInt64()); VersionMadeBy = reader.ReadUInt16(); VersionNeededToExtract = reader.ReadUInt16(); VolumeNumber = reader.ReadUInt32(); @@ -18,12 +25,42 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader TotalNumberOfEntries = (long)reader.ReadUInt64(); DirectorySize = (long)reader.ReadUInt64(); DirectoryStartOffsetRelativeToDisk = (long)reader.ReadUInt64(); - DataSector = reader.ReadBytes( - (int)( - SizeOfDirectoryEndRecord - - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS - ) - ); + // Extensible data is not interpreted. Consume it without allocating from its declared size. + var remaining = + SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS; + if (remaining == 0) + { + return; + } + var buffer = _pool.Rent(4096); + try + { + while (remaining > 0) + { + var read = reader.Read(buffer, 0, (int)Math.Min(4096, remaining)); + if (read == 0) + { + throw new IncompleteArchiveException("Truncated ZIP64 extensible data."); + } + remaining -= read; + } + } + finally + { + _pool.Return(buffer); + } + } + + private static long ValidateRecordSize(ulong size) + { + if ( + size < SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS + || size > long.MaxValue + ) + { + throw new InvalidFormatException("Invalid ZIP64 end record size."); + } + return (long)size; } private const int SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS = 44; @@ -45,6 +82,4 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader public long DirectorySize { get; private set; } public long DirectoryStartOffsetRelativeToDisk { get; private set; } - - public byte[]? DataSector { get; private set; } } diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs index 56496c9a..96d6da7d 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs @@ -1,8 +1,10 @@ using System; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.ArcLzw; @@ -15,37 +17,19 @@ public partial class ArcLzwStream CancellationToken cancellationToken ) { - if (_processed) + cancellationToken.ThrowIfCancellationRequested(); + if (count == 0) { return 0; } - _processed = true; - var data = new byte[_compressedSize]; - int totalRead = 0; - while (totalRead < _compressedSize) + if (!_processed) { - int read = await _stream - .ReadAsync(data, totalRead, _compressedSize - totalRead, cancellationToken) - .ConfigureAwait(false); - if (read == 0) - { - break; - } - totalRead += read; + using var limited = new ReadOnlySubStream(_stream, _compressedSize); + using var data = new PooledMemoryStream(); + await limited.CopyToAsync(data, 81920, cancellationToken).ConfigureAwait(false); + FinishDecompression(data); } - var decoded = Decompress(data, _useCrunched); - var result = decoded.Count; - if (_useCrunched) - { - var unpacked = RLE.UnpackRLE(decoded.ToArray()); - unpacked.CopyTo(buffer, 0); - result = unpacked.Count; - } - else - { - decoded.CopyTo(buffer, 0); - } - return result; + return Read(buffer, offset, count); } #if !LEGACY_DOTNET diff --git a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs index 542d991a..10a32111 100644 --- a/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs +++ b/src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using SharpCompress.Common; using SharpCompress.Compressors.RLE90; +using SharpCompress.IO; namespace SharpCompress.Compressors.ArcLzw; @@ -13,6 +14,8 @@ public partial class ArcLzwStream : Stream private bool _processed; private bool _useCrunched; private int _compressedSize; + private byte[]? _decoded; + private int _decodedPosition; private const int BITS = 12; private const int CRUNCH_BITS = 12; @@ -34,6 +37,10 @@ public partial class ArcLzwStream : Stream public ArcLzwStream(Stream stream, int compressedSize, bool useCrunched = true) { + if (compressedSize < 0) + { + throw new InvalidFormatException("ARC: invalid compressed size."); + } _stream = stream; _useCrunched = useCrunched; _compressedSize = compressedSize; @@ -177,28 +184,35 @@ public partial class ArcLzwStream : Stream public override int Read(byte[] buffer, int offset, int count) { - if (_processed) + if (count == 0) { return 0; } - _processed = true; - var data = new byte[_compressedSize]; - _stream.Read(data, 0, _compressedSize); - var decoded = Decompress(data, _useCrunched); - var result = decoded.Count; - if (_useCrunched) + if (!_processed) { - var unpacked = RLE.UnpackRLE(decoded.ToArray()); - unpacked.CopyTo(buffer, 0); - result = unpacked.Count; - } - else - { - decoded.CopyTo(buffer, 0); + using var limited = new ReadOnlySubStream(_stream, _compressedSize); + using var data = new PooledMemoryStream(); + limited.CopyTo(data); + FinishDecompression(data); } + var result = Math.Min(count, _decoded!.Length - _decodedPosition); + Buffer.BlockCopy(_decoded, _decodedPosition, buffer, offset, result); + _decodedPosition += result; return result; } + private void FinishDecompression(PooledMemoryStream data) + { + if (data.Length != _compressedSize) + { + throw new IncompleteArchiveException("Truncated ARC compressed data."); + } + // Buffer only bytes actually read, never the size claimed by the ARC header. + var decoded = Decompress(data.ToArray(), _useCrunched); + _decoded = _useCrunched ? RLE.UnpackRLE(decoded.ToArray()).ToArray() : decoded.ToArray(); + _processed = true; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs index 6c7f541f..a6bbdfff 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs @@ -95,6 +95,7 @@ internal partial class OutWindow : IAsyncDisposable var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize); if (fastSize >= 2) { + EnsureCapacity(_pos + fastSize); _buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize)); _pos += fastSize; pos += fastSize; @@ -120,6 +121,7 @@ internal partial class OutWindow : IAsyncDisposable public async ValueTask PutByteAsync(byte b, CancellationToken cancellationToken = default) { + EnsureCapacity(_pos + 1); _buffer[_pos++] = b; _total++; if (_pos >= _windowSize) @@ -139,7 +141,7 @@ internal partial class OutWindow : IAsyncDisposable { cancellationToken.ThrowIfCancellationRequested(); - var curSize = _windowSize - _pos; + var curSize = Math.Min(4096, _windowSize - _pos); if (curSize > _limit - _total) { curSize = (int)(_limit - _total); @@ -148,6 +150,7 @@ internal partial class OutWindow : IAsyncDisposable { curSize = size; } + EnsureCapacity(_pos + curSize); var numReadBytes = await stream .ReadAsync(_buffer, _pos, curSize, cancellationToken) .ConfigureAwait(false); @@ -173,7 +176,7 @@ internal partial class OutWindow : IAsyncDisposable stream.Position = len - size; _total = 0; _limit = size; - _pos = _windowSize - size; + _pos = 0; await CopyStreamAsync(stream, size).ConfigureAwait(false); if (_pos == _windowSize) { diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs index ef79fcdc..4cf13d5a 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs @@ -31,23 +31,24 @@ internal partial class OutWindow : IDisposable public long Total => _total; + internal int BufferCapacity => _buffer?.Length ?? 0; + public void Create(int windowSize) { if (windowSize <= 0) { throw new InvalidFormatException($"LZMA: invalid dictionary size {windowSize}"); } - if (_windowSize != windowSize) + if (_windowSize != windowSize || _buffer is null) { // Allocate first so failed growth leaves the existing buffer owned by this window. - var replacement = _pool.Rent(windowSize); + var replacement = _pool.Rent(Math.Min(windowSize, 4096)); if (_buffer is not null) { _pool.Return(_buffer); } _buffer = replacement; } - _buffer[windowSize - 1] = 0; _windowSize = windowSize; _pos = 0; _streamPos = 0; @@ -56,6 +57,20 @@ internal partial class OutWindow : IDisposable _limit = 0; } + private void EnsureCapacity(int required) + { + if (required <= _buffer.Length) + { + return; + } + // Grow from emitted history, never from the advertised dictionary or output size. + var capacity = (int)Math.Min(_windowSize, Math.Max(required, (long)_buffer.Length * 2)); + var replacement = _pool.Rent(capacity); + _buffer.AsSpan(0, _pos).CopyTo(replacement); + _pool.Return(_buffer); + _buffer = replacement; + } + public void Dispose() { try @@ -93,7 +108,7 @@ internal partial class OutWindow : IDisposable stream.Position = len - size; _total = 0; _limit = size; - _pos = _windowSize - size; + _pos = 0; CopyStream(stream, size); if (_pos == _windowSize) { @@ -157,6 +172,7 @@ internal partial class OutWindow : IDisposable var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize); if (fastSize >= 2) { + EnsureCapacity(_pos + fastSize); _buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize)); _pos += fastSize; pos += fastSize; @@ -182,6 +198,7 @@ internal partial class OutWindow : IDisposable public void PutByte(byte b) { + EnsureCapacity(_pos + 1); _buffer[_pos++] = b; _total++; if (_pos >= _windowSize) @@ -192,6 +209,11 @@ internal partial class OutWindow : IDisposable public byte GetByte(int distance) { + // The initial literal context is zero; no dictionary history has been emitted yet. + if (_total == 0 && distance == 0) + { + return 0; + } var pos = _pos - distance - 1; if (pos < 0) { @@ -205,7 +227,7 @@ internal partial class OutWindow : IDisposable var size = len; while (size > 0 && _pos < _windowSize && _total < _limit) { - var curSize = _windowSize - _pos; + var curSize = Math.Min(4096, _windowSize - _pos); if (curSize > _limit - _total) { curSize = (int)(_limit - _total); @@ -214,6 +236,7 @@ internal partial class OutWindow : IDisposable { curSize = size; } + EnsureCapacity(_pos + curSize); var numReadBytes = stream.Read(_buffer, _pos, curSize); if (numReadBytes == 0) { diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index cdb0c91f..24ce5fd6 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -210,6 +210,11 @@ public partial class LzmaStream } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -233,6 +238,10 @@ public partial class LzmaStream } var read = _outWindow.Read(buffer, offset, toProcess); + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; @@ -327,6 +336,11 @@ public partial class LzmaStream } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -350,6 +364,10 @@ public partial class LzmaStream } var read = _outWindow.Read(buffer, offset, toProcess); + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index d2a22a4e..361378a6 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -65,6 +65,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } else { + // Larger standard dictionaries exceed this decoder's positive Int32 window range. + if (properties[0] > 37) + { + throw new InvalidParamException(); + } _dictionarySize = 2 | (properties[0] & 1); _dictionarySize <<= (properties[0] >> 1) + 11; @@ -318,6 +323,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } } + if (_availableBytes < 0) + { + throw new DataErrorException(); + } + var toProcess = count - total; if (toProcess > _availableBytes) { @@ -335,6 +345,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } var read = _outWindow.Read(buffer, offset, toProcess); + // Unknown-size raw LZMA can end without output; otherwise no progress is corruption. + if (read == 0 && _availableBytes > 0) + { + throw new DataErrorException(); + } total += read; offset += read; _position += read; @@ -469,10 +484,21 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable return value; } + private int ReadChunkHeaderByte() + { + var value = _inputStream!.ReadByte(); + // EOF is not an LZMA2 end marker and must never enter chunk-size arithmetic. + if (value < 0) + { + throw new DataErrorException(); + } + _inputPosition++; + return value; + } + private void DecodeChunkHeader() { - var control = _inputStream!.ReadByte(); - _inputPosition++; + var control = ReadChunkHeaderByte(); if (control == 0x00) { @@ -501,17 +527,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable _uncompressedChunk = false; _availableBytes = (control & 0x1F) << 16; - _availableBytes += (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _availableBytes += (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; - _rangeDecoderLimit = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _rangeDecoderLimit = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; if (control >= 0xC0) { _needProps = false; - Properties[0] = (byte)_inputStream.ReadByte(); - _inputPosition++; + Properties[0] = (byte)ReadChunkHeaderByte(); _decoder = new Decoder(); _decoder.SetDecoderProperties(Properties); @@ -535,8 +558,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable else { _uncompressedChunk = true; - _availableBytes = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; + _availableBytes = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1; } } diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index 744682c4..d2a166bd 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -210,7 +210,8 @@ public class PpmdStream : Stream, IAsyncDisposable _isDisposed = true; if (_compress) { - await _model.EncodeBlockAsync(_stream, new MemoryStream(), true).ConfigureAwait(false); + using var input = new PooledMemoryStream(); + await _model.EncodeBlockAsync(_stream, input, true).ConfigureAwait(false); } _modelH?.Dispose(); _modelH = null; diff --git a/src/SharpCompress/Compressors/Shrink/BitStream.cs b/src/SharpCompress/Compressors/Shrink/BitStream.cs index 77df3fa3..84841662 100644 --- a/src/SharpCompress/Compressors/Shrink/BitStream.cs +++ b/src/SharpCompress/Compressors/Shrink/BitStream.cs @@ -9,6 +9,7 @@ internal class BitStream private int _bitIdx; private int _bitsLeft; private ulong _bitBuffer; + private bool _lastReadValid; private static uint[] _maskBits = new uint[17] { 0U, @@ -62,6 +63,11 @@ internal class BitStream _bitsLeft += 8; } } + _lastReadValid = _bitsLeft >= nbits; + if (!_lastReadValid) + { + return 0; + } result = (int)(_bitBuffer & _maskBits[nbits]); _bitBuffer >>= nbits; _bitsLeft -= nbits; @@ -70,10 +76,6 @@ internal class BitStream public bool Advance(int count) { - if (_byteIdx > _srcLen) - { - return false; - } - return true; + return _lastReadValid; } } diff --git a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs index 81e411c1..706674bc 100644 --- a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs +++ b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; namespace SharpCompress.Compressors.Shrink; @@ -238,6 +239,63 @@ public class HwUnshrink byte[] dst, int dstCap, out int dstUsed + ) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, null); + + internal static UnshrnkStatus UnshrinkGrowing( + byte[] src, + int srcLen, + int dstCap, + ArrayPool pool, + out byte[] dst, + out int dstUsed + ) + { + dst = Array.Empty(); + try + { + return UnshrinkCore(src, srcLen, out _, ref dst, dstCap, out dstUsed, pool); + } + catch + { + if (dst.Length > 0) + { + pool.Return(dst); + dst = Array.Empty(); + } + throw; + } + } + + private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, ArrayPool? pool) + { + if (pool is null) + { + return; + } + // A code string is bounded by the code table's ushort length. Reserve at most + // one string ahead of actual output, not the size advertised by the ZIP header. + var required = (int)Math.Min(dstCap, (long)dstPos + ushort.MaxValue + 1); + if (required > dst.Length) + { + var capacity = (int)Math.Min(dstCap, Math.Max(required, (long)dst.Length * 2)); + var replacement = pool.Rent(capacity); + dst.AsSpan(0, dstPos).CopyTo(replacement); + if (dst.Length > 0) + { + pool.Return(dst); + } + dst = replacement; + } + } + + private static UnshrnkStatus UnshrinkCore( + byte[] src, + int srcLen, + out int srcUsed, + ref byte[] dst, + int dstCap, + out int dstUsed, + ArrayPool? pool ) { var codeTab = new CodeTabEntry[HASHTAB_SIZE]; @@ -279,6 +337,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, pool); firstByte = (byte)currCode; dst[dstPos] = (byte)currCode; codeTab[currCode].lastDstPos = dstPos; @@ -301,6 +360,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, pool); // Handle KwKwK: next code used before being added. if (currCode == queue.codes[queue.nextIdx]) { diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs index c5f8c49a..5185c401 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Compressors.Shrink; @@ -28,15 +29,10 @@ internal partial class ShrinkStream : Stream // Read actual compressed data from the stream rather than pre-allocating based on the // declared compressed size, which may be crafted to cause an OutOfMemoryException. // The stream is already bounded by ReadOnlySubStream in ZipFilePart. - using var srcMs = new MemoryStream(); + using var srcMs = new PooledMemoryStream(); await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false); - var src = srcMs.ToArray(); - var srcLen = src.Length; - - // Decompress synchronously (CPU-bound operation) - HwUnshrink.Unshrink(src, srcLen, out _, _byteOut, (int)_uncompressedSize, out var dstUsed); - _outBytesCount = dstUsed; - _decompressed = true; + // Decompress synchronously (CPU-bound operation). + Decode(srcMs); } public override async Task ReadAsync( @@ -46,6 +42,7 @@ internal partial class ShrinkStream : Stream CancellationToken cancellationToken ) { + ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); if (!_decompressed) @@ -72,6 +69,7 @@ internal partial class ShrinkStream : Stream CancellationToken cancellationToken = default ) { + ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); if (!_decompressed) diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs index a3e680b2..1a6d69e5 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs @@ -1,22 +1,29 @@ using System; +using System.Buffers; using System.IO; using SharpCompress.Common; +using SharpCompress.IO; namespace SharpCompress.Compressors.Shrink; internal partial class ShrinkStream : Stream { private readonly Stream _inStream; + private readonly ArrayPool _pool; private readonly long _uncompressedSize; - private readonly byte[] _byteOut; + private byte[] _byteOut = Array.Empty(); private long _outBytesCount; private bool _decompressed; + private bool _disposed; private long _position; public ShrinkStream(Stream stream, long uncompressedSize) + : this(stream, uncompressedSize, ArrayPool.Shared) { } + + internal ShrinkStream(Stream stream, long uncompressedSize, ArrayPool pool) { - if (uncompressedSize > int.MaxValue) + if (uncompressedSize < 0 || uncompressedSize > int.MaxValue) { throw new InvalidFormatException( $"Shrink: declared uncompressed size {uncompressedSize} exceeds maximum supported size." @@ -24,13 +31,13 @@ internal partial class ShrinkStream : Stream } _inStream = stream; + _pool = pool; _uncompressedSize = uncompressedSize; - _byteOut = new byte[(int)_uncompressedSize]; _outBytesCount = 0L; } - public override bool CanRead => true; + public override bool CanRead => !_disposed; public override bool CanSeek => true; @@ -48,27 +55,15 @@ internal partial class ShrinkStream : Stream public override int Read(byte[] buffer, int offset, int count) { + ThrowIfDisposed(); if (!_decompressed) { // Read actual compressed data from the stream rather than pre-allocating based on the // declared compressed size, which may be crafted to cause an OutOfMemoryException. // The stream is already bounded by ReadOnlySubStream in ZipFilePart. - using var srcMs = new MemoryStream(); + using var srcMs = new PooledMemoryStream(); _inStream.CopyTo(srcMs); - var src = srcMs.ToArray(); - var srcLen = src.Length; - - HwUnshrink.Unshrink( - src, - srcLen, - out _, - _byteOut, - (int)_uncompressedSize, - out var dstUsed - ); - _outBytesCount = dstUsed; - _decompressed = true; - _position = 0; + Decode(srcMs); } long remaining = _outBytesCount - _position; @@ -86,6 +81,70 @@ internal partial class ShrinkStream : Stream public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + private void Decode(PooledMemoryStream source) + { + var length = checked((int)source.Length); + var src = _pool.Rent(length); + var output = Array.Empty(); + try + { + source.Position = 0; + if (source.Read(src, 0, length) != length) + { + throw new InvalidFormatException("Shrink: truncated compressed data."); + } + // A pool rental may be larger than the compressed data; never decode its unused tail. + var status = HwUnshrink.UnshrinkGrowing( + src, + length, + (int)_uncompressedSize, + _pool, + out output, + out var used + ); + if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize) + { + throw new InvalidFormatException("Shrink: invalid or truncated compressed data."); + } + _byteOut = output; + output = Array.Empty(); + _outBytesCount = used; + _decompressed = true; + _position = 0; + } + finally + { + _pool.Return(src); + if (output.Length > 0) + { + _pool.Return(output); + } + } + } + + protected override void Dispose(bool disposing) + { + if (!_disposed) + { + _disposed = true; + var output = _byteOut; + _byteOut = Array.Empty(); + if (output.Length > 0) + { + _pool.Return(output); + } + } + base.Dispose(disposing); + } + + private void ThrowIfDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ShrinkStream)); + } + } + public override void SetLength(long value) => throw new NotImplementedException(); public override void Write(byte[] buffer, int offset, int count) => diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs index 9cb57f6a..4d61dfad 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs @@ -55,7 +55,7 @@ public partial class SevenZipWriter var filesInfo = new SevenZipFilesInfoWriter { Entries = entries.ToArray() }; // Write header to a temporary stream first - using var headerStream = new MemoryStream(); + using var headerStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteRawHeader(headerStream, mainStreamsInfo, filesInfo); // Optionally compress the header @@ -70,7 +70,7 @@ public partial class SevenZipWriter } private async ValueTask WriteCompressedHeaderAsync( - MemoryStream rawHeaderStream, + PooledMemoryStream rawHeaderStream, long endOfPackedData ) { @@ -97,7 +97,7 @@ public partial class SevenZipWriter }; // Write encoded header to a second temporary stream - using var encodedHeaderStream = new MemoryStream(); + using var encodedHeaderStream = new PooledMemoryStream(); ArchiveHeaderWriter.WriteEncodedHeader(encodedHeaderStream, headerStreamsInfo); // Write the encoded header to the output @@ -125,7 +125,7 @@ public partial class SevenZipWriter } private async ValueTask WriteRawHeaderToOutputAsync( - MemoryStream rawHeaderStream, + PooledMemoryStream rawHeaderStream, long endOfPackedData ) { diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs index b934c6e1..fbf5ab07 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; +using SharpCompress.IO; namespace SharpCompress.Writers.Zip; @@ -24,7 +25,7 @@ public partial class ZipWriter // Buffer the entire central directory + end record into memory, then write async. // This avoids synchronous writes to the underlying stream during finalization. - using var ms = new MemoryStream(); + using var ms = new PooledMemoryStream(); ulong size = 0; foreach (var entry in entries) { @@ -149,9 +150,9 @@ public partial class ZipWriter CancellationToken cancellationToken ) { - // Build the header synchronously into a MemoryStream, then async-copy to OutputStream. + // Build the header synchronously into a pooled stream, then async-copy to OutputStream. // This avoids any synchronous writes to the potentially async-only output stream. - using var ms = new MemoryStream(); + using var ms = new PooledMemoryStream(); var outputStream = OutputStream.NotNull(); var result = WriteHeader( ms, diff --git a/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs b/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs new file mode 100644 index 00000000..9c16694e --- /dev/null +++ b/tests/SharpCompress.Test/Arc/ArcAllocationTests.cs @@ -0,0 +1,58 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.ArcLzw; +using Xunit; + +namespace SharpCompress.Test.Arc; + +public class ArcAllocationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Arc_Rejects_Huge_Truncated_Compressed_Data(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 12, 65, 0 }); + using var stream = new ArcLzwStream(input, int.MaxValue); + var buffer = new byte[1]; + if (useAsync) + { + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => stream.Read(buffer, 0, 1)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Arc_Preserves_Output_Across_Small_Reads(bool useAsync) + { + // Two nine-bit literals, followed by a byte belonging to the next entry. + using var input = new MemoryStream(new byte[] { 65, 132, 0, 99 }); + using var stream = new ArcLzwStream(input, 3, false); + var buffer = new byte[3]; + foreach (var expected in new byte[] { 65, 66 }) + { + var read = useAsync + ? await stream.ReadAsync(buffer, 1, 1, CancellationToken.None) + : stream.Read(buffer, 1, 1); + Assert.Equal(1, read); + Assert.Equal(expected, buffer[1]); + Assert.Equal(0, buffer[0]); + } + Assert.Equal(0, stream.Read(buffer, 0, 1)); + Assert.Equal(99, input.ReadByte()); + } + + [Fact] + public void Arc_Rejects_Negative_Compressed_Size() => + Assert.Throws(() => new ArcLzwStream(Stream.Null, -1)); +} diff --git a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs index dd26ba01..874cca29 100644 --- a/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs +++ b/tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs @@ -6,9 +6,11 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Tar.Headers; +using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.LZMA.LZ; using SharpCompress.Compressors.LZMA.Utilities; +using SharpCompress.Compressors.Shrink; using SharpCompress.IO; using Xunit; @@ -16,6 +18,184 @@ namespace SharpCompress.Test.Streams; public class BufferPoolLifetimeTests { + [Theory] + [InlineData(false, 0, false)] + [InlineData(true, 0, false)] + [InlineData(false, 10000, false)] + [InlineData(true, 10000, false)] + [InlineData(false, 1, true)] + [InlineData(true, 1, true)] + public async Task Zip64_ExtensibleData_ReturnsBuffer( + bool useAsync, + int payloadSize, + bool truncated + ) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(); + using (var writer = new BinaryWriter(input, System.Text.Encoding.UTF8, true)) + { + writer.Write((ulong)(44 + payloadSize + (truncated ? 10000 : 0))); + writer.Write(new byte[44 + payloadSize]); + if (!truncated) + { + writer.Write((byte)99); + } + } + input.Position = 0; + var header = new Zip64DirectoryEndHeader(pool); + if (useAsync) + { + using var reader = new AsyncBinaryReader(input, leaveOpen: true); + if (truncated) + { + await Assert.ThrowsAnyAsync(async () => + await header.Read(reader) + ); + } + else + { + await header.Read(reader); + } + } + else + { + using var reader = new BinaryReader(input, System.Text.Encoding.UTF8, true); + if (truncated) + { + Assert.ThrowsAny(() => header.Read(reader)); + } + else + { + header.Read(reader); + } + } + if (!truncated) + { + Assert.Equal(99, input.ReadByte()); + } + Assert.Equal(payloadSize == 0 ? 0 : 1, pool.RentCount); + Assert.Equal(pool.RentCount, pool.ReturnCount); + Assert.Empty(pool.Active); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shrink_Dispose_ReturnsBuffers(bool useAsync, bool asyncDispose) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(new byte[] { 65, 132, 0 }); + using var stream = new ShrinkStream(input, 2, pool); + var buffer = new byte[1]; + foreach (var expected in new byte[] { 65, 66 }) + { + var read = useAsync + ? await stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + : stream.Read(buffer, 0, 1); + Assert.Equal(1, read); + Assert.Equal(expected, buffer[0]); + } + Assert.Equal(0, stream.Read(buffer, 0, 1)); + Assert.Equal(2, pool.RentCount); + Assert.Equal(1, pool.ReturnCount); + Assert.Single(pool.Active); + +#if !LEGACY_DOTNET + if (asyncDispose) + { + await stream.DisposeAsync(); + } + else +#endif + { + stream.Dispose(); + } + stream.Dispose(); + Assert.Equal(2, pool.ReturnCount); + Assert.Empty(pool.Active); + Assert.True(input.CanRead); + Assert.False(stream.CanRead); + Assert.Throws(() => stream.Read(buffer, 0, 1)); + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Shrink_InvalidData_ReturnsBuffers(bool useAsync) + { + var pool = new TrackingPool { ExtraCapacity = 32 }; + using var input = new MemoryStream(new byte[] { 65, 132, 0 }); + using var stream = new ShrinkStream(input, int.MaxValue, pool); + var buffer = new byte[1]; + if (useAsync) + { + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, 1, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => stream.Read(buffer, 0, 1)); + } + Assert.Equal(2, pool.RentCount); + Assert.Equal(2, pool.ReturnCount); + Assert.Empty(pool.Active); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Shrink_Growth_ReturnsBuffers(bool failGrowth) + { + // Literal-only nine-bit codes fill the first output rental without increasing code size. + const int count = 70000; + var input = new byte[(count * 9 + 7) / 8]; + for (var i = 0; i < count; i++) + { + var bit = i * 9; + input[bit / 8] |= (byte)(65 << (bit % 8)); + input[bit / 8 + 1] |= (byte)(65 >> (8 - bit % 8)); + } + var pool = new TrackingPool { FailRentNumber = failGrowth ? 2 : 0 }; + if (failGrowth) + { + Assert.Throws(() => + HwUnshrink.UnshrinkGrowing(input, input.Length, count, pool, out _, out _) + ); + Assert.Equal(1, pool.ReturnCount); + } + else + { + var status = HwUnshrink.UnshrinkGrowing( + input, + input.Length, + count, + pool, + out var output, + out var used + ); + try + { + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(count, used); + Assert.All(output, value => Assert.Equal(65, value)); + Assert.Equal(1, pool.ReturnCount); + } + finally + { + pool.Return(output); + } + Assert.Equal(2, pool.ReturnCount); + } + Assert.Empty(pool.Active); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -101,6 +281,74 @@ public class BufferPoolLifetimeTests Assert.Empty(pool.Active); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task OutWindow_LazyGrowth_PreservesPoolOwnership(bool asyncWrite, bool failGrowth) + { + var pool = new TrackingPool(); + using var window = new OutWindow(pool); + window.Create(16384); + Assert.Equal(4096, window.BufferCapacity); + + for (var i = 0; i < 4096; i++) + { + if (asyncWrite) + { + await window.PutByteAsync(42, CancellationToken.None); + } + else + { + window.PutByte(42); + } + } + + if (failGrowth) + { + pool.FailNextRent = true; + if (asyncWrite) + { + await Assert.ThrowsAsync(async () => + await window.PutByteAsync(43, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => window.PutByte(43)); + } + Assert.Equal(0, pool.ReturnCount); + Assert.Equal(4096, window.BufferCapacity); + Assert.Equal(42, window.GetByte(0)); + } + + if (asyncWrite) + { + await window.PutByteAsync(43, CancellationToken.None); + } + else + { + window.PutByte(43); + } + Assert.Equal(8192, window.BufferCapacity); + Assert.Equal(42, window.GetByte(4096)); + Assert.Equal(43, window.GetByte(0)); + Assert.Equal(1, pool.ReturnCount); + Assert.Single(pool.Active); + + if (asyncWrite) + { + await window.DisposeAsync(); + } + else + { + window.Dispose(); + } + Assert.Equal(2, pool.ReturnCount); + Assert.Empty(pool.Active); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -302,18 +550,20 @@ public class BufferPoolLifetimeTests internal int RentCount { get; private set; } internal int ReturnCount { get; private set; } internal bool FailNextRent { get; set; } + internal int FailRentNumber { get; set; } + internal int ExtraCapacity { get; set; } public override T[] Rent(int minimumLength) { RentCount++; - if (FailNextRent) + if (FailNextRent || RentCount == FailRentNumber) { FailNextRent = false; #pragma warning disable CA2201 // Simulate allocation failure without exhausting process memory. throw new OutOfMemoryException("Injected pool allocation failure."); #pragma warning restore CA2201 } - var buffer = new T[minimumLength]; + var buffer = new T[minimumLength + ExtraCapacity]; Assert.True(Active.Add(buffer)); return buffer; } diff --git a/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs new file mode 100644 index 00000000..4fa16d41 --- /dev/null +++ b/tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.Xz; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class Lzma2TruncationTests +{ + private const string TruncatedXzHex = + "FD377A585A0000016922DE360200210100000000372797D6010011616C7068610A616C7068610A616C7068610A"; + private const string MutatedSevenZipBase64 = + "N3q8ryccAATrtqxIDgAAAAAAAABaAAAAAAAAAKyqq98BAApoZWxsbyB3b3JsZAEEBgABCQ4ABwsBAAEhIQEADAwACAoBhRFKDQAABQEZDAAAAAAAAAAAAAAAABETAHQAZQBzAHQALgB0AHgAdAAAABkAFAoBAI6gjYxMV90BFQYBACAAAAAAAA=="; + + public static IEnumerable PartialHeaders() + { + // Start with a complete dictionary-reset chunk so a missing control byte + // cannot be rejected merely because the dictionary still needs resetting. + byte[] prefix = [0x01, 0x00, 0x00, 0x58]; + yield return [prefix]; + byte[][] headers = + [ + [0x01, 0x00, 0x00], + [0x02, 0x00, 0x00], + [0xE0, 0x00, 0x00, 0x00, 0x04, 0x00], + ]; + foreach (var header in headers) + { + for (var length = 1; length < header.Length; length++) + { + var data = new byte[prefix.Length + length]; + prefix.CopyTo(data, 0); + Array.Copy(header, 0, data, prefix.Length, length); + yield return [data]; + } + } + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public void Lzma2_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + using var decoder = LzmaStream.Create([0x00], input); + Assert.ThrowsAny(() => decoder.CopyTo(Stream.Null)); + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public void Lzma2_ReadByte_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + using var decoder = LzmaStream.Create([0x00], input); + Assert.Equal(0x58, decoder.ReadByte()); + Assert.ThrowsAny(() => decoder.ReadByte()); + } + + [Theory] + [MemberData(nameof(PartialHeaders))] + public async Task Lzma2_Async_Rejects_Partial_Header(byte[] data) + { + using var input = new MemoryStream(data); + await using var decoder = await LzmaStream.CreateAsync([0x00], input, -1, -1); + var buffer = new byte[64]; + await Assert.ThrowsAnyAsync(async () => + await decoder.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None) + ); +#if !LEGACY_DOTNET + // Use a fresh decoder because parser state is undefined after a failed read. + using var memoryInput = new MemoryStream(data); + await using var memoryDecoder = await LzmaStream.CreateAsync([0x00], memoryInput, -1, -1); + await Assert.ThrowsAnyAsync(async () => + await memoryDecoder.ReadAsync(buffer.AsMemory(), CancellationToken.None) + ); +#endif + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Xz_Rejects_Missing_Lzma2_End_Marker(bool useAsync) + { + var data = new byte[TruncatedXzHex.Length / 2]; + for (var i = 0; i < data.Length; i++) + { + data[i] = Convert.ToByte(TruncatedXzHex.Substring(i * 2, 2), 16); + } + using var input = new MemoryStream(data); + await using var decoder = new XZStream(input); + if (useAsync) + { + await Assert.ThrowsAnyAsync(() => + decoder.CopyToAsync(Stream.Null, 81920, CancellationToken.None) + ); + } + else + { + Assert.ThrowsAny(() => decoder.CopyTo(Stream.Null)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SevenZip_Rejects_Missing_Lzma2_End_Marker(bool useAsync) + { + using var input = new MemoryStream(Convert.FromBase64String(MutatedSevenZipBase64)); + if (useAsync) + { + await using var archive = await ArchiveFactory.OpenAsyncArchive(input); + await foreach (var entry in archive.EntriesAsync) + { + await using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None); + await Assert.ThrowsAnyAsync(() => + stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None) + ); + } + } + else + { + using var archive = ArchiveFactory.OpenArchive(input); + foreach (var entry in archive.Entries) + { + using var stream = entry.OpenEntryStream(); + Assert.ThrowsAny(() => stream.CopyTo(Stream.Null)); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma2_Accepts_End_Marker_After_Uncompressed_Chunks(bool useAsync) + { + byte[] data = [0x01, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x59, 0x00]; + using var input = new MemoryStream(data); + using var output = new MemoryStream(); + if (useAsync) + { + await using var decoder = await LzmaStream.CreateAsync([0x00], input, data.Length, 2); + await decoder.CopyToAsync(output, 1, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create([0x00], input, data.Length, 2); + decoder.CopyTo(output, 1); + } + Assert.Equal(new byte[] { 0x58, 0x59 }, output.ToArray()); + } +} diff --git a/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs b/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs new file mode 100644 index 00000000..6e8d536f --- /dev/null +++ b/tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs @@ -0,0 +1,226 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.LZMA.LZ; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class LzmaDictionaryAllocationTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Lzma_Raw_Huge_Dictionary_Decodes_With_Stream_And_Direct_APIs( + bool useAsync, + bool direct + ) + { + var expected = new byte[20000]; + new Random(123).NextBytes(expected); + using var packed = new MemoryStream(); + byte[] properties; + using ( + var encoder = LzmaStream.Create(new LzmaEncoderProperties(false, 4096), false, packed) + ) + { + properties = (byte[])encoder.Properties.Clone(); + encoder.Write(expected, 0, expected.Length); + } + // The compressed payload needs only a small dictionary, regardless of its advertised size. + properties[1] = 255; + properties[2] = 255; + properties[3] = 255; + properties[4] = 127; + using var input = new MemoryStream(packed.ToArray()); + using var output = new MemoryStream(); + if (direct) + { + using var decoder = new Decoder(); + decoder.SetDecoderProperties(properties); + if (useAsync) + { + await decoder.CodeAsync( + input, + output, + input.Length, + expected.Length, + null!, + CancellationToken.None + ); + } + else + { + decoder.Code(input, output, input.Length, expected.Length, null!); + } + } + else if (useAsync) + { + using var decoder = await LzmaStream.CreateAsync( + properties, + input, + input.Length, + expected.Length + ); + await decoder.CopyToAsync(output, 137, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create(properties, input, input.Length, expected.Length); + decoder.CopyTo(output, 137); + } + Assert.Equal(expected, output.ToArray()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_Truncated_Uncompressed_Chunk_Does_Not_Allocate_From_Read_Size( + bool useAsync + ) + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + window.SetLimit(int.MaxValue); + using var input = new MemoryStream(new byte[] { 1 }); + if (useAsync) + { + await Assert.ThrowsAnyAsync(async () => + await window.CopyStreamAsync(input, int.MaxValue, CancellationToken.None) + ); + } + else + { + Assert.ThrowsAny(() => window.CopyStream(input, int.MaxValue)); + } + Assert.InRange(window.BufferCapacity, 1, 8192); + } + + [Theory] + [InlineData(38)] + [InlineData(40)] + [InlineData(64)] + [InlineData(255)] + public void Lzma2_Rejects_Unsupported_Dictionary_Properties(int property) + { + Assert.ThrowsAny(() => + LzmaStream.Create(new byte[] { (byte)property }, Stream.Null) + ); + } + + [Fact] + public void Lzma_Huge_Dictionary_Starts_Small_And_Grows_With_History() + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + Assert.InRange(window.BufferCapacity, 1, 4096); + Assert.Equal(0, window.GetByte(0)); + window.SetLimit(20000); + for (var i = 0; i < 20000; i++) + { + window.PutByte((byte)i); + } + Assert.InRange(window.BufferCapacity, 20000, 32768); + for (var i = 0; i < 20000; i++) + { + Assert.Equal((byte)i, window.GetByte(19999 - i)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_History_Survives_Growth_Wrap_And_Pending_Matches(bool useAsync) + { + using var window = new OutWindow(); + window.Create(8192); + var output = new byte[8192]; + window.SetLimit(8192); + for (var i = 0; i < output.Length; i++) + { + if (useAsync) + { + await window.PutByteAsync((byte)i, CancellationToken.None); + } + else + { + window.PutByte((byte)i); + } + } + Assert.Equal(output.Length, window.Read(output, 0, output.Length)); + window.SetLimit(10); + if (useAsync) + { + await window.CopyBlockAsync(8191, 20, CancellationToken.None); + } + else + { + window.CopyBlock(8191, 20); + } + Assert.True(window.HasPending); + window.SetLimit(10); + if (useAsync) + { + await window.CopyPendingAsync(CancellationToken.None); + } + else + { + window.CopyPending(); + } + Assert.False(window.HasPending); + Assert.Equal(20, window.Read(output, 0, output.Length)); + for (var i = 0; i < 20; i++) + { + Assert.Equal((byte)i, output[i]); + } + window.Reset(); + Assert.Equal(0, window.Total); + Assert.Equal(0, window.GetByte(0)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma_Preset_Dictionary_Does_Not_Allocate_Declared_Window(bool useAsync) + { + using var window = new OutWindow(); + window.Create(int.MaxValue); + using var preset = new MemoryStream(new byte[] { 1, 2, 3 }); + if (useAsync) + { + await window.TrainAsync(preset); + } + else + { + window.Train(preset); + } + Assert.Equal(3, window.GetByte(0)); + Assert.Equal(1, window.GetByte(2)); + Assert.InRange(window.BufferCapacity, 3, 4096); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Lzma2_Tiny_Stream_With_Huge_Dictionary_Decodes(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 }); + using var output = new MemoryStream(); + if (useAsync) + { + using var decoder = await LzmaStream.CreateAsync(new byte[] { 37 }, input, -1, -1); + await decoder.CopyToAsync(output, 81920, CancellationToken.None); + } + else + { + using var decoder = LzmaStream.Create(new byte[] { 37 }, input); + decoder.CopyTo(output); + } + Assert.Equal(new byte[] { 65 }, output.ToArray()); + } +} diff --git a/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs new file mode 100644 index 00000000..4fd281de --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs @@ -0,0 +1,74 @@ +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.Shrink; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class ShrinkAllocationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Shrink_Rejects_Huge_Truncated_Output(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 65, 0 }); + if (useAsync) + { + await Assert.ThrowsAsync(async () => + { + using var stream = await ShrinkStream.CreateAsync( + input, + int.MaxValue, + CancellationToken.None + ); + }); + } + else + { + using var stream = new ShrinkStream(input, int.MaxValue); + Assert.Throws(() => stream.ReadByte()); + } + } + + [Fact] + public void Shrink_Grows_Only_For_Actual_Output() + { + var pool = ArrayPool.Shared; + var status = HwUnshrink.UnshrinkGrowing( + new byte[] { 65, 132, 0 }, + 3, + int.MaxValue, + pool, + out var output, + out var used + ); + try + { + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(2, used); + Assert.Equal(65, output[0]); + Assert.Equal(66, output[1]); + Assert.InRange(output.Length, 2, 131072); + } + finally + { + pool.Return(output); + } + } + + [Fact] + public void Shrink_Preserves_Fixed_Buffer_API() + { + var output = new byte[2]; + Assert.Equal( + HwUnshrink.UnshrnkStatus.Ok, + HwUnshrink.Unshrink(new byte[] { 65, 132, 0 }, 3, out _, output, 2, out var used) + ); + Assert.Equal(2, used); + Assert.Equal(new byte[] { 65, 66 }, output); + } +} diff --git a/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs b/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs new file mode 100644 index 00000000..48ed6024 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Common.Zip.Headers; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class Zip64AllocationTests +{ + [Theory] + [InlineData(0UL)] + [InlineData(43UL)] + [InlineData(ulong.MaxValue)] + public async Task Zip64_Rejects_Invalid_Record_Size(ulong size) + { + using var stream = CreateRecord(size, 0); + Assert.Throws(() => + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)) + ); + stream.Position = 0; + await Assert.ThrowsAsync(async () => + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)) + ); + } + + [Theory] + [InlineData(44UL, 0)] + [InlineData(10044UL, 10000)] + public async Task Zip64_Skips_Extensible_Data_Exactly(ulong size, int payloadSize) + { + using var stream = CreateRecord(size, payloadSize); + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)); + Assert.Equal(stream.Length, stream.Position); + stream.Position = 0; + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)); + Assert.Equal(stream.Length, stream.Position); + } + + [Theory] + [InlineData(2147483691UL)] + [InlineData(4294967340UL)] + public async Task Zip64_Rejects_Huge_Truncated_Extensible_Data(ulong size) + { + using var stream = CreateRecord(size, 1); + Assert.ThrowsAny(() => + new Zip64DirectoryEndHeader().Read(new BinaryReader(stream)) + ); + stream.Position = 0; + await Assert.ThrowsAnyAsync(async () => + await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream)) + ); + } + + private static MemoryStream CreateRecord(ulong size, int payloadSize) + { + var stream = new MemoryStream(); + using (var writer = new BinaryWriter(stream, System.Text.Encoding.UTF8, true)) + { + writer.Write(size); + writer.Write(new byte[44 + payloadSize]); + } + stream.Position = 0; + return stream; + } +}