diff --git a/docs/USAGE.md b/docs/USAGE.md index d78ae51e..511ac6b4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -477,3 +477,5 @@ await archive.WriteToDirectoryAsync( ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size. ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size. + +ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected. diff --git a/src/SharpCompress/Compressors/Shrink/BitStream.cs b/src/SharpCompress/Compressors/Shrink/BitStream.cs index 77df3fa3..84841662 100644 --- a/src/SharpCompress/Compressors/Shrink/BitStream.cs +++ b/src/SharpCompress/Compressors/Shrink/BitStream.cs @@ -9,6 +9,7 @@ internal class BitStream private int _bitIdx; private int _bitsLeft; private ulong _bitBuffer; + private bool _lastReadValid; private static uint[] _maskBits = new uint[17] { 0U, @@ -62,6 +63,11 @@ internal class BitStream _bitsLeft += 8; } } + _lastReadValid = _bitsLeft >= nbits; + if (!_lastReadValid) + { + return 0; + } result = (int)(_bitBuffer & _maskBits[nbits]); _bitBuffer >>= nbits; _bitsLeft -= nbits; @@ -70,10 +76,6 @@ internal class BitStream public bool Advance(int count) { - if (_byteIdx > _srcLen) - { - return false; - } - return true; + return _lastReadValid; } } diff --git a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs index 81e411c1..25170093 100644 --- a/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs +++ b/src/SharpCompress/Compressors/Shrink/HwUnshrink.cs @@ -238,6 +238,43 @@ public class HwUnshrink byte[] dst, int dstCap, out int dstUsed + ) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, false); + + internal static UnshrnkStatus UnshrinkGrowing( + byte[] src, + int dstCap, + out byte[] dst, + out int dstUsed + ) + { + dst = Array.Empty(); + return UnshrinkCore(src, src.Length, out _, ref dst, dstCap, out dstUsed, true); + } + + private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, bool grow) + { + if (!grow) + { + return; + } + // A code string is bounded by the code table's ushort length. Reserve at most + // one string ahead of actual output, not the size advertised by the ZIP header. + var required = (int)Math.Min(dstCap, (long)dstPos + ushort.MaxValue + 1); + if (required > dst.Length) + { + var capacity = (int)Math.Min(dstCap, Math.Max(required, (long)dst.Length * 2)); + Array.Resize(ref dst, capacity); + } + } + + private static UnshrnkStatus UnshrinkCore( + byte[] src, + int srcLen, + out int srcUsed, + ref byte[] dst, + int dstCap, + out int dstUsed, + bool grow ) { var codeTab = new CodeTabEntry[HASHTAB_SIZE]; @@ -279,6 +316,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, grow); firstByte = (byte)currCode; dst[dstPos] = (byte)currCode; codeTab[currCode].lastDstPos = dstPos; @@ -301,6 +339,7 @@ public class HwUnshrink return UnshrnkStatus.Full; } + ReserveOutput(ref dst, dstPos, dstCap, grow); // Handle KwKwK: next code used before being added. if (currCode == queue.codes[queue.nextIdx]) { diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs index c5f8c49a..49895df3 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs @@ -31,12 +31,8 @@ internal partial class ShrinkStream : Stream using var srcMs = new MemoryStream(); await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false); var src = srcMs.ToArray(); - var srcLen = src.Length; - - // Decompress synchronously (CPU-bound operation) - HwUnshrink.Unshrink(src, srcLen, out _, _byteOut, (int)_uncompressedSize, out var dstUsed); - _outBytesCount = dstUsed; - _decompressed = true; + // Decompress synchronously (CPU-bound operation). + Decode(src); } public override async Task ReadAsync( diff --git a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs index a3e680b2..a1f34259 100644 --- a/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs +++ b/src/SharpCompress/Compressors/Shrink/ShrinkStream.cs @@ -9,14 +9,14 @@ internal partial class ShrinkStream : Stream private readonly Stream _inStream; private readonly long _uncompressedSize; - private readonly byte[] _byteOut; + private byte[] _byteOut = Array.Empty(); private long _outBytesCount; private bool _decompressed; private long _position; public ShrinkStream(Stream stream, long uncompressedSize) { - if (uncompressedSize > int.MaxValue) + if (uncompressedSize < 0 || uncompressedSize > int.MaxValue) { throw new InvalidFormatException( $"Shrink: declared uncompressed size {uncompressedSize} exceeds maximum supported size." @@ -26,7 +26,6 @@ internal partial class ShrinkStream : Stream _inStream = stream; _uncompressedSize = uncompressedSize; - _byteOut = new byte[(int)_uncompressedSize]; _outBytesCount = 0L; } @@ -56,19 +55,7 @@ internal partial class ShrinkStream : Stream using var srcMs = new MemoryStream(); _inStream.CopyTo(srcMs); var src = srcMs.ToArray(); - var srcLen = src.Length; - - HwUnshrink.Unshrink( - src, - srcLen, - out _, - _byteOut, - (int)_uncompressedSize, - out var dstUsed - ); - _outBytesCount = dstUsed; - _decompressed = true; - _position = 0; + Decode(src); } long remaining = _outBytesCount - _position; @@ -86,6 +73,24 @@ internal partial class ShrinkStream : Stream public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + private void Decode(byte[] src) + { + var status = HwUnshrink.UnshrinkGrowing( + src, + (int)_uncompressedSize, + out var output, + out var used + ); + if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize) + { + throw new InvalidFormatException("Shrink: invalid or truncated compressed data."); + } + _byteOut = output; + _outBytesCount = used; + _decompressed = true; + _position = 0; + } + public override void SetLength(long value) => throw new NotImplementedException(); public override void Write(byte[] buffer, int offset, int count) => diff --git a/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs new file mode 100644 index 00000000..31e0e317 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs @@ -0,0 +1,63 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.Shrink; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class ShrinkAllocationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Shrink_Rejects_Huge_Truncated_Output(bool useAsync) + { + using var input = new MemoryStream(new byte[] { 65, 0 }); + if (useAsync) + { + await Assert.ThrowsAsync(async () => + { + using var stream = await ShrinkStream.CreateAsync( + input, + int.MaxValue, + CancellationToken.None + ); + }); + } + else + { + using var stream = new ShrinkStream(input, int.MaxValue); + Assert.Throws(() => stream.ReadByte()); + } + } + + [Fact] + public void Shrink_Grows_Only_For_Actual_Output() + { + var status = HwUnshrink.UnshrinkGrowing( + new byte[] { 65, 132, 0 }, + int.MaxValue, + out var output, + out var used + ); + Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status); + Assert.Equal(2, used); + Assert.Equal(65, output[0]); + Assert.Equal(66, output[1]); + Assert.InRange(output.Length, 2, 131072); + } + + [Fact] + public void Shrink_Preserves_Fixed_Buffer_API() + { + var output = new byte[2]; + Assert.Equal( + HwUnshrink.UnshrnkStatus.Ok, + HwUnshrink.Unshrink(new byte[] { 65, 132, 0 }, 3, out _, output, 2, out var used) + ); + Assert.Equal(2, used); + Assert.Equal(new byte[] { 65, 66 }, output); + } +}