starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-11-15 17:02:03 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Merge commit from fork

Browse Source 
Reject truncated LZMA2 headers and prevent stalled decoding
This commit is contained in:
Adam Hathcock
2026-10-19 10:27:41 +01:00
committed by GitHub
parent cd9ce9ad94 7642f76f38
commit 2fe5f3a4e8
24 changed files with 1206 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/USAGE.md
View File

@@ -472,3 +472,12 @@ await archive.WriteToDirectoryAsync(
- Support for cancellation via CancellationToken
- Better resource utilization in async/await contexts
- Compatible with modern .NET async patterns
## Malformed input and memory use
ZIP64 end-record extensible data is skipped with bounded buffering. Invalid record sizes and truncated extensible data are rejected rather than driving allocations from the declared size.
ARC LZW compressed data is buffered incrementally from the bytes actually present. A truncated entry is rejected without preallocating its declared compressed size.
ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected.
LZMA/LZMA2 dictionary storage starts small and grows with decoded history, up to the supported dictionary size. These safeguards avoid allocations based solely on forged sizes; they do not impose an overall decompressed-output or process-memory limit.

6
src/SharpCompress/Common/Arc/ArcFilePart.Async.cs
View File

@@ -37,6 +37,12 @@ public partial class ArcFilePart
.ConfigureAwait(false);
break;
case CompressionType.Crunched:
if (Header.CompressedSize > int.MaxValue)
{
throw new InvalidFormatException(
"ARC: compressed size exceeds the supported range."
);
}
if (Header.OriginalSize > 128 * 1024)
{
throw new NotSupportedException(

6
src/SharpCompress/Common/Arc/ArcFilePart.cs
View File

@@ -52,6 +52,12 @@ public partial class ArcFilePart : FilePart
compressedStream = SqueezeStream.Create(_stream, (int)Header.CompressedSize);
break;
case CompressionType.Crunched:
if (Header.CompressedSize > int.MaxValue)
{
throw new InvalidFormatException(
"ARC: compressed size exceeds the supported range."
);
}
if (Header.OriginalSize > 128 * 1024)
{
throw new NotSupportedException(

30
src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.Async.cs
View File

@@ -1,3 +1,4 @@
using System;
using System.IO;
using System.Threading.Tasks;
using SharpCompress.IO;
@@ -8,7 +9,9 @@ internal partial class Zip64DirectoryEndHeader
{
internal override async ValueTask Read(AsyncBinaryReader reader)
{
SizeOfDirectoryEndRecord = (long)await reader.ReadUInt64Async().ConfigureAwait(false);
SizeOfDirectoryEndRecord = ValidateRecordSize(
await reader.ReadUInt64Async().ConfigureAwait(false)
);
VersionMadeBy = await reader.ReadUInt16Async().ConfigureAwait(false);
VersionNeededToExtract = await reader.ReadUInt16Async().ConfigureAwait(false);
VolumeNumber = await reader.ReadUInt32Async().ConfigureAwait(false);
@@ -18,10 +21,25 @@ internal partial class Zip64DirectoryEndHeader
DirectorySize = (long)await reader.ReadUInt64Async().ConfigureAwait(false);
DirectoryStartOffsetRelativeToDisk = (long)
await reader.ReadUInt64Async().ConfigureAwait(false);
var size = (int)(
SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS
);
DataSector = new byte[size];
await reader.ReadBytesAsync(DataSector, 0, size).ConfigureAwait(false);
var remaining =
SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS;
if (remaining == 0)
{
return;
}
var buffer = _pool.Rent(4096);
try
{
while (remaining > 0)
{
var size = (int)Math.Min(4096, remaining);
await reader.ReadBytesAsync(buffer, 0, size).ConfigureAwait(false);
remaining -= size;
}
}
finally
{
_pool.Return(buffer);
}
}
}

55
src/SharpCompress/Common/Zip/Headers/Zip64DirectoryEndHeader.cs
View File

@@ -1,15 +1,22 @@
using System;
using System.Buffers;
using System.IO;
namespace SharpCompress.Common.Zip.Headers;
internal partial class Zip64DirectoryEndHeader : ZipHeader
{
private readonly ArrayPool<byte> _pool;
public Zip64DirectoryEndHeader()
: base(ZipHeaderType.Zip64DirectoryEnd) { }
: this(ArrayPool<byte>.Shared) { }
internal Zip64DirectoryEndHeader(ArrayPool<byte> pool)
: base(ZipHeaderType.Zip64DirectoryEnd) => _pool = pool;
internal override void Read(BinaryReader reader)
{
SizeOfDirectoryEndRecord = (long)reader.ReadUInt64();
SizeOfDirectoryEndRecord = ValidateRecordSize(reader.ReadUInt64());
VersionMadeBy = reader.ReadUInt16();
VersionNeededToExtract = reader.ReadUInt16();
VolumeNumber = reader.ReadUInt32();
@@ -18,12 +25,42 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader
TotalNumberOfEntries = (long)reader.ReadUInt64();
DirectorySize = (long)reader.ReadUInt64();
DirectoryStartOffsetRelativeToDisk = (long)reader.ReadUInt64();
DataSector = reader.ReadBytes(
(int)(
SizeOfDirectoryEndRecord
- SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS
)
);
// Extensible data is not interpreted. Consume it without allocating from its declared size.
var remaining =
SizeOfDirectoryEndRecord - SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS;
if (remaining == 0)
{
return;
}
var buffer = _pool.Rent(4096);
try
{
while (remaining > 0)
{
var read = reader.Read(buffer, 0, (int)Math.Min(4096, remaining));
if (read == 0)
{
throw new IncompleteArchiveException("Truncated ZIP64 extensible data.");
}
remaining -= read;
}
}
finally
{
_pool.Return(buffer);
}
}
private static long ValidateRecordSize(ulong size)
{
if (
size < SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS
|| size > long.MaxValue
)
{
throw new InvalidFormatException("Invalid ZIP64 end record size.");
}
return (long)size;
}
private const int SIZE_OF_FIXED_HEADER_DATA_EXCEPT_SIGNATURE_AND_SIZE_FIELDS = 44;
@@ -45,6 +82,4 @@ internal partial class Zip64DirectoryEndHeader : ZipHeader
public long DirectorySize { get; private set; }
public long DirectoryStartOffsetRelativeToDisk { get; private set; }
public byte[]? DataSector { get; private set; }
}

36
src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.Async.cs
View File

@@ -1,8 +1,10 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.RLE90;
using SharpCompress.IO;
namespace SharpCompress.Compressors.ArcLzw;
@@ -15,37 +17,19 @@ public partial class ArcLzwStream
CancellationToken cancellationToken
)
{
if (_processed)
cancellationToken.ThrowIfCancellationRequested();
if (count == 0)
{
return 0;
}
_processed = true;
var data = new byte[_compressedSize];
int totalRead = 0;
while (totalRead < _compressedSize)
if (!_processed)
{
int read = await _stream
.ReadAsync(data, totalRead, _compressedSize - totalRead, cancellationToken)
.ConfigureAwait(false);
if (read == 0)
{
break;
}
totalRead += read;
using var limited = new ReadOnlySubStream(_stream, _compressedSize);
using var data = new PooledMemoryStream();
await limited.CopyToAsync(data, 81920, cancellationToken).ConfigureAwait(false);
FinishDecompression(data);
}
var decoded = Decompress(data, _useCrunched);
var result = decoded.Count;
if (_useCrunched)
{
var unpacked = RLE.UnpackRLE(decoded.ToArray());
unpacked.CopyTo(buffer, 0);
result = unpacked.Count;
}
else
{
decoded.CopyTo(buffer, 0);
}
return result;
return Read(buffer, offset, count);
}
#if !LEGACY_DOTNET

42
src/SharpCompress/Compressors/ArcLzw/ArcLzwStream.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using SharpCompress.Common;
using SharpCompress.Compressors.RLE90;
using SharpCompress.IO;
namespace SharpCompress.Compressors.ArcLzw;
@@ -13,6 +14,8 @@ public partial class ArcLzwStream : Stream
private bool _processed;
private bool _useCrunched;
private int _compressedSize;
private byte[]? _decoded;
private int _decodedPosition;
private const int BITS = 12;
private const int CRUNCH_BITS = 12;
@@ -34,6 +37,10 @@ public partial class ArcLzwStream : Stream
public ArcLzwStream(Stream stream, int compressedSize, bool useCrunched = true)
{
if (compressedSize < 0)
{
throw new InvalidFormatException("ARC: invalid compressed size.");
}
_stream = stream;
_useCrunched = useCrunched;
_compressedSize = compressedSize;
@@ -177,28 +184,35 @@ public partial class ArcLzwStream : Stream
public override int Read(byte[] buffer, int offset, int count)
{
if (_processed)
if (count == 0)
{
return 0;
}
_processed = true;
var data = new byte[_compressedSize];
_stream.Read(data, 0, _compressedSize);
var decoded = Decompress(data, _useCrunched);
var result = decoded.Count;
if (_useCrunched)
if (!_processed)
{
var unpacked = RLE.UnpackRLE(decoded.ToArray());
unpacked.CopyTo(buffer, 0);
result = unpacked.Count;
}
else
{
decoded.CopyTo(buffer, 0);
using var limited = new ReadOnlySubStream(_stream, _compressedSize);
using var data = new PooledMemoryStream();
limited.CopyTo(data);
FinishDecompression(data);
}
var result = Math.Min(count, _decoded!.Length - _decodedPosition);
Buffer.BlockCopy(_decoded, _decodedPosition, buffer, offset, result);
_decodedPosition += result;
return result;
}
private void FinishDecompression(PooledMemoryStream data)
{
if (data.Length != _compressedSize)
{
throw new IncompleteArchiveException("Truncated ARC compressed data.");
}
// Buffer only bytes actually read, never the size claimed by the ARC header.
var decoded = Decompress(data.ToArray(), _useCrunched);
_decoded = _useCrunched ? RLE.UnpackRLE(decoded.ToArray()).ToArray() : decoded.ToArray();
_processed = true;
}
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();

7
src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.Async.cs
View File

@@ -95,6 +95,7 @@ internal partial class OutWindow : IAsyncDisposable
var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize);
if (fastSize >= 2)
{
EnsureCapacity(_pos + fastSize);
_buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize));
_pos += fastSize;
pos += fastSize;
@@ -120,6 +121,7 @@ internal partial class OutWindow : IAsyncDisposable
public async ValueTask PutByteAsync(byte b, CancellationToken cancellationToken = default)
{
EnsureCapacity(_pos + 1);
_buffer[_pos++] = b;
_total++;
if (_pos >= _windowSize)
@@ -139,7 +141,7 @@ internal partial class OutWindow : IAsyncDisposable
{
cancellationToken.ThrowIfCancellationRequested();
var curSize = _windowSize - _pos;
var curSize = Math.Min(4096, _windowSize - _pos);
if (curSize > _limit - _total)
{
curSize = (int)(_limit - _total);
@@ -148,6 +150,7 @@ internal partial class OutWindow : IAsyncDisposable
{
curSize = size;
}
EnsureCapacity(_pos + curSize);
var numReadBytes = await stream
.ReadAsync(_buffer, _pos, curSize, cancellationToken)
.ConfigureAwait(false);
@@ -173,7 +176,7 @@ internal partial class OutWindow : IAsyncDisposable
stream.Position = len - size;
_total = 0;
_limit = size;
_pos = _windowSize - size;
_pos = 0;
await CopyStreamAsync(stream, size).ConfigureAwait(false);
if (_pos == _windowSize)
{

33
src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs
View File

@@ -31,23 +31,24 @@ internal partial class OutWindow : IDisposable
public long Total => _total;
internal int BufferCapacity => _buffer?.Length ?? 0;
public void Create(int windowSize)
{
if (windowSize <= 0)
{
throw new InvalidFormatException($"LZMA: invalid dictionary size {windowSize}");
}
if (_windowSize != windowSize)
if (_windowSize != windowSize || _buffer is null)
{
// Allocate first so failed growth leaves the existing buffer owned by this window.
var replacement = _pool.Rent(windowSize);
var replacement = _pool.Rent(Math.Min(windowSize, 4096));
if (_buffer is not null)
{
_pool.Return(_buffer);
}
_buffer = replacement;
}
_buffer[windowSize - 1] = 0;
_windowSize = windowSize;
_pos = 0;
_streamPos = 0;
@@ -56,6 +57,20 @@ internal partial class OutWindow : IDisposable
_limit = 0;
}
private void EnsureCapacity(int required)
{
if (required <= _buffer.Length)
{
return;
}
// Grow from emitted history, never from the advertised dictionary or output size.
var capacity = (int)Math.Min(_windowSize, Math.Max(required, (long)_buffer.Length * 2));
var replacement = _pool.Rent(capacity);
_buffer.AsSpan(0, _pos).CopyTo(replacement);
_pool.Return(_buffer);
_buffer = replacement;
}
public void Dispose()
{
try
@@ -93,7 +108,7 @@ internal partial class OutWindow : IDisposable
stream.Position = len - size;
_total = 0;
_limit = size;
_pos = _windowSize - size;
_pos = 0;
CopyStream(stream, size);
if (_pos == _windowSize)
{
@@ -157,6 +172,7 @@ internal partial class OutWindow : IDisposable
var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize);
if (fastSize >= 2)
{
EnsureCapacity(_pos + fastSize);
_buffer.AsSpan(pos, fastSize).CopyTo(_buffer.AsSpan(_pos, fastSize));
_pos += fastSize;
pos += fastSize;
@@ -182,6 +198,7 @@ internal partial class OutWindow : IDisposable
public void PutByte(byte b)
{
EnsureCapacity(_pos + 1);
_buffer[_pos++] = b;
_total++;
if (_pos >= _windowSize)
@@ -192,6 +209,11 @@ internal partial class OutWindow : IDisposable
public byte GetByte(int distance)
{
// The initial literal context is zero; no dictionary history has been emitted yet.
if (_total == 0 && distance == 0)
{
return 0;
}
var pos = _pos - distance - 1;
if (pos < 0)
{
@@ -205,7 +227,7 @@ internal partial class OutWindow : IDisposable
var size = len;
while (size > 0 && _pos < _windowSize && _total < _limit)
{
var curSize = _windowSize - _pos;
var curSize = Math.Min(4096, _windowSize - _pos);
if (curSize > _limit - _total)
{
curSize = (int)(_limit - _total);
@@ -214,6 +236,7 @@ internal partial class OutWindow : IDisposable
{
curSize = size;
}
EnsureCapacity(_pos + curSize);
var numReadBytes = stream.Read(_buffer, _pos, curSize);
if (numReadBytes == 0)
{

18
src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs
View File

@@ -210,6 +210,11 @@ public partial class LzmaStream
}
}
if (_availableBytes < 0)
{
throw new DataErrorException();
}
var toProcess = count - total;
if (toProcess > _availableBytes)
{
@@ -233,6 +238,10 @@ public partial class LzmaStream
}
var read = _outWindow.Read(buffer, offset, toProcess);
if (read == 0 && _availableBytes > 0)
{
throw new DataErrorException();
}
total += read;
offset += read;
_position += read;
@@ -327,6 +336,11 @@ public partial class LzmaStream
}
}
if (_availableBytes < 0)
{
throw new DataErrorException();
}
var toProcess = count - total;
if (toProcess > _availableBytes)
{
@@ -350,6 +364,10 @@ public partial class LzmaStream
}
var read = _outWindow.Read(buffer, offset, toProcess);
if (read == 0 && _availableBytes > 0)
{
throw new DataErrorException();
}
total += read;
offset += read;
_position += read;

42
src/SharpCompress/Compressors/LZMA/LzmaStream.cs
View File

@@ -65,6 +65,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
else
{
// Larger standard dictionaries exceed this decoder's positive Int32 window range.
if (properties[0] > 37)
{
throw new InvalidParamException();
}
_dictionarySize = 2 | (properties[0] & 1);
_dictionarySize <<= (properties[0] >> 1) + 11;
@@ -318,6 +323,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
}
if (_availableBytes < 0)
{
throw new DataErrorException();
}
var toProcess = count - total;
if (toProcess > _availableBytes)
{
@@ -335,6 +345,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
var read = _outWindow.Read(buffer, offset, toProcess);
// Unknown-size raw LZMA can end without output; otherwise no progress is corruption.
if (read == 0 && _availableBytes > 0)
{
throw new DataErrorException();
}
total += read;
offset += read;
_position += read;
@@ -469,10 +484,21 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
return value;
}
private int ReadChunkHeaderByte()
{
var value = _inputStream!.ReadByte();
// EOF is not an LZMA2 end marker and must never enter chunk-size arithmetic.
if (value < 0)
{
throw new DataErrorException();
}
_inputPosition++;
return value;
}
private void DecodeChunkHeader()
{
var control = _inputStream!.ReadByte();
_inputPosition++;
var control = ReadChunkHeaderByte();
if (control == 0x00)
{
@@ -501,17 +527,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
_uncompressedChunk = false;
_availableBytes = (control & 0x1F) << 16;
_availableBytes += (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1;
_inputPosition += 2;
_availableBytes += (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1;
_rangeDecoderLimit = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1;
_inputPosition += 2;
_rangeDecoderLimit = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1;
if (control >= 0xC0)
{
_needProps = false;
Properties[0] = (byte)_inputStream.ReadByte();
_inputPosition++;
Properties[0] = (byte)ReadChunkHeaderByte();
_decoder = new Decoder();
_decoder.SetDecoderProperties(Properties);
@@ -535,8 +558,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
else
{
_uncompressedChunk = true;
_availableBytes = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1;
_inputPosition += 2;
_availableBytes = (ReadChunkHeaderByte() << 8) + ReadChunkHeaderByte() + 1;
}
}

3
src/SharpCompress/Compressors/PPMd/PpmdStream.cs
View File

@@ -210,7 +210,8 @@ public class PpmdStream : Stream, IAsyncDisposable
_isDisposed = true;
if (_compress)
{
await _model.EncodeBlockAsync(_stream, new MemoryStream(), true).ConfigureAwait(false);
using var input = new PooledMemoryStream();
await _model.EncodeBlockAsync(_stream, input, true).ConfigureAwait(false);
}
_modelH?.Dispose();
_modelH = null;

12
src/SharpCompress/Compressors/Shrink/BitStream.cs
View File

@@ -9,6 +9,7 @@ internal class BitStream
private int _bitIdx;
private int _bitsLeft;
private ulong _bitBuffer;
private bool _lastReadValid;
private static uint[] _maskBits = new uint[17]
{
0U,
@@ -62,6 +63,11 @@ internal class BitStream
_bitsLeft += 8;
}
}
_lastReadValid = _bitsLeft >= nbits;
if (!_lastReadValid)
{
return 0;
}
result = (int)(_bitBuffer & _maskBits[nbits]);
_bitBuffer >>= nbits;
_bitsLeft -= nbits;
@@ -70,10 +76,6 @@ internal class BitStream
public bool Advance(int count)
{
if (_byteIdx > _srcLen)
{
return false;
}
return true;
return _lastReadValid;
}
}

60
src/SharpCompress/Compressors/Shrink/HwUnshrink.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Buffers;
namespace SharpCompress.Compressors.Shrink;
@@ -238,6 +239,63 @@ public class HwUnshrink
byte[] dst,
int dstCap,
out int dstUsed
) => UnshrinkCore(src, srcLen, out srcUsed, ref dst, dstCap, out dstUsed, null);
internal static UnshrnkStatus UnshrinkGrowing(
byte[] src,
int srcLen,
int dstCap,
ArrayPool<byte> pool,
out byte[] dst,
out int dstUsed
)
{
dst = Array.Empty<byte>();
try
{
return UnshrinkCore(src, srcLen, out _, ref dst, dstCap, out dstUsed, pool);
}
catch
{
if (dst.Length > 0)
{
pool.Return(dst);
dst = Array.Empty<byte>();
}
throw;
}
}
private static void ReserveOutput(ref byte[] dst, int dstPos, int dstCap, ArrayPool<byte>? pool)
{
if (pool is null)
{
return;
}
// A code string is bounded by the code table's ushort length. Reserve at most
// one string ahead of actual output, not the size advertised by the ZIP header.
var required = (int)Math.Min(dstCap, (long)dstPos + ushort.MaxValue + 1);
if (required > dst.Length)
{
var capacity = (int)Math.Min(dstCap, Math.Max(required, (long)dst.Length * 2));
var replacement = pool.Rent(capacity);
dst.AsSpan(0, dstPos).CopyTo(replacement);
if (dst.Length > 0)
{
pool.Return(dst);
}
dst = replacement;
}
}
private static UnshrnkStatus UnshrinkCore(
byte[] src,
int srcLen,
out int srcUsed,
ref byte[] dst,
int dstCap,
out int dstUsed,
ArrayPool<byte>? pool
)
{
var codeTab = new CodeTabEntry[HASHTAB_SIZE];
@@ -279,6 +337,7 @@ public class HwUnshrink
return UnshrnkStatus.Full;
}
ReserveOutput(ref dst, dstPos, dstCap, pool);
firstByte = (byte)currCode;
dst[dstPos] = (byte)currCode;
codeTab[currCode].lastDstPos = dstPos;
@@ -301,6 +360,7 @@ public class HwUnshrink
return UnshrnkStatus.Full;
}
ReserveOutput(ref dst, dstPos, dstCap, pool);
// Handle KwKwK: next code used before being added.
if (currCode == queue.codes[queue.nextIdx])
{

14
src/SharpCompress/Compressors/Shrink/ShrinkStream.Async.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Shrink;
@@ -28,15 +29,10 @@ internal partial class ShrinkStream : Stream
// Read actual compressed data from the stream rather than pre-allocating based on the
// declared compressed size, which may be crafted to cause an OutOfMemoryException.
// The stream is already bounded by ReadOnlySubStream in ZipFilePart.
using var srcMs = new MemoryStream();
using var srcMs = new PooledMemoryStream();
await _inStream.CopyToAsync(srcMs, 81920, cancellationToken).ConfigureAwait(false);
var src = srcMs.ToArray();
var srcLen = src.Length;
// Decompress synchronously (CPU-bound operation)
HwUnshrink.Unshrink(src, srcLen, out _, _byteOut, (int)_uncompressedSize, out var dstUsed);
_outBytesCount = dstUsed;
_decompressed = true;
// Decompress synchronously (CPU-bound operation).
Decode(srcMs);
}
public override async Task<int> ReadAsync(
@@ -46,6 +42,7 @@ internal partial class ShrinkStream : Stream
CancellationToken cancellationToken
)
{
ThrowIfDisposed();
cancellationToken.ThrowIfCancellationRequested();
if (!_decompressed)
@@ -72,6 +69,7 @@ internal partial class ShrinkStream : Stream
CancellationToken cancellationToken = default
)
{
ThrowIfDisposed();
cancellationToken.ThrowIfCancellationRequested();
if (!_decompressed)

97
src/SharpCompress/Compressors/Shrink/ShrinkStream.cs
View File

@@ -1,22 +1,29 @@
using System;
using System.Buffers;
using System.IO;
using SharpCompress.Common;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Shrink;
internal partial class ShrinkStream : Stream
{
private readonly Stream _inStream;
private readonly ArrayPool<byte> _pool;
private readonly long _uncompressedSize;
private readonly byte[] _byteOut;
private byte[] _byteOut = Array.Empty<byte>();
private long _outBytesCount;
private bool _decompressed;
private bool _disposed;
private long _position;
public ShrinkStream(Stream stream, long uncompressedSize)
: this(stream, uncompressedSize, ArrayPool<byte>.Shared) { }
internal ShrinkStream(Stream stream, long uncompressedSize, ArrayPool<byte> pool)
{
if (uncompressedSize > int.MaxValue)
if (uncompressedSize < 0 || uncompressedSize > int.MaxValue)
{
throw new InvalidFormatException(
$"Shrink: declared uncompressed size {uncompressedSize} exceeds maximum supported size."
@@ -24,13 +31,13 @@ internal partial class ShrinkStream : Stream
}
_inStream = stream;
_pool = pool;
_uncompressedSize = uncompressedSize;
_byteOut = new byte[(int)_uncompressedSize];
_outBytesCount = 0L;
}
public override bool CanRead => true;
public override bool CanRead => !_disposed;
public override bool CanSeek => true;
@@ -48,27 +55,15 @@ internal partial class ShrinkStream : Stream
public override int Read(byte[] buffer, int offset, int count)
{
ThrowIfDisposed();
if (!_decompressed)
{
// Read actual compressed data from the stream rather than pre-allocating based on the
// declared compressed size, which may be crafted to cause an OutOfMemoryException.
// The stream is already bounded by ReadOnlySubStream in ZipFilePart.
using var srcMs = new MemoryStream();
using var srcMs = new PooledMemoryStream();
_inStream.CopyTo(srcMs);
var src = srcMs.ToArray();
var srcLen = src.Length;
HwUnshrink.Unshrink(
src,
srcLen,
out _,
_byteOut,
(int)_uncompressedSize,
out var dstUsed
);
_outBytesCount = dstUsed;
_decompressed = true;
_position = 0;
Decode(srcMs);
}
long remaining = _outBytesCount - _position;
@@ -86,6 +81,70 @@ internal partial class ShrinkStream : Stream
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();
private void Decode(PooledMemoryStream source)
{
var length = checked((int)source.Length);
var src = _pool.Rent(length);
var output = Array.Empty<byte>();
try
{
source.Position = 0;
if (source.Read(src, 0, length) != length)
{
throw new InvalidFormatException("Shrink: truncated compressed data.");
}
// A pool rental may be larger than the compressed data; never decode its unused tail.
var status = HwUnshrink.UnshrinkGrowing(
src,
length,
(int)_uncompressedSize,
_pool,
out output,
out var used
);
if (status != HwUnshrink.UnshrnkStatus.Ok || used != _uncompressedSize)
{
throw new InvalidFormatException("Shrink: invalid or truncated compressed data.");
}
_byteOut = output;
output = Array.Empty<byte>();
_outBytesCount = used;
_decompressed = true;
_position = 0;
}
finally
{
_pool.Return(src);
if (output.Length > 0)
{
_pool.Return(output);
}
}
}
protected override void Dispose(bool disposing)
{
if (!_disposed)
{
_disposed = true;
var output = _byteOut;
_byteOut = Array.Empty<byte>();
if (output.Length > 0)
{
_pool.Return(output);
}
}
base.Dispose(disposing);
}
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(ShrinkStream));
}
}
public override void SetLength(long value) => throw new NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) =>

8
src/SharpCompress/Writers/SevenZip/SevenZipWriter.Async.cs
View File

@@ -55,7 +55,7 @@ public partial class SevenZipWriter
var filesInfo = new SevenZipFilesInfoWriter { Entries = entries.ToArray() };
// Write header to a temporary stream first
using var headerStream = new MemoryStream();
using var headerStream = new PooledMemoryStream();
ArchiveHeaderWriter.WriteRawHeader(headerStream, mainStreamsInfo, filesInfo);
// Optionally compress the header
@@ -70,7 +70,7 @@ public partial class SevenZipWriter
}
private async ValueTask WriteCompressedHeaderAsync(
MemoryStream rawHeaderStream,
PooledMemoryStream rawHeaderStream,
long endOfPackedData
)
{
@@ -97,7 +97,7 @@ public partial class SevenZipWriter
};
// Write encoded header to a second temporary stream
using var encodedHeaderStream = new MemoryStream();
using var encodedHeaderStream = new PooledMemoryStream();
ArchiveHeaderWriter.WriteEncodedHeader(encodedHeaderStream, headerStreamsInfo);
// Write the encoded header to the output
@@ -125,7 +125,7 @@ public partial class SevenZipWriter
}
private async ValueTask WriteRawHeaderToOutputAsync(
MemoryStream rawHeaderStream,
PooledMemoryStream rawHeaderStream,
long endOfPackedData
)
{

7
src/SharpCompress/Writers/Zip/ZipWriter.Async.cs
View File

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
namespace SharpCompress.Writers.Zip;
@@ -24,7 +25,7 @@ public partial class ZipWriter
// Buffer the entire central directory + end record into memory, then write async.
// This avoids synchronous writes to the underlying stream during finalization.
using var ms = new MemoryStream();
using var ms = new PooledMemoryStream();
ulong size = 0;
foreach (var entry in entries)
{
@@ -149,9 +150,9 @@ public partial class ZipWriter
CancellationToken cancellationToken
)
{
// Build the header synchronously into a MemoryStream, then async-copy to OutputStream.
// Build the header synchronously into a pooled stream, then async-copy to OutputStream.
// This avoids any synchronous writes to the potentially async-only output stream.
using var ms = new MemoryStream();
using var ms = new PooledMemoryStream();
var outputStream = OutputStream.NotNull();
var result = WriteHeader(
ms,

58
tests/SharpCompress.Test/Arc/ArcAllocationTests.cs Normal file
View File

@@ -0,0 +1,58 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.ArcLzw;
using Xunit;
namespace SharpCompress.Test.Arc;
public class ArcAllocationTests
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Arc_Rejects_Huge_Truncated_Compressed_Data(bool useAsync)
{
using var input = new MemoryStream(new byte[] { 12, 65, 0 });
using var stream = new ArcLzwStream(input, int.MaxValue);
var buffer = new byte[1];
if (useAsync)
{
await Assert.ThrowsAsync<IncompleteArchiveException>(() =>
stream.ReadAsync(buffer, 0, 1, CancellationToken.None)
);
}
else
{
Assert.Throws<IncompleteArchiveException>(() => stream.Read(buffer, 0, 1));
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Arc_Preserves_Output_Across_Small_Reads(bool useAsync)
{
// Two nine-bit literals, followed by a byte belonging to the next entry.
using var input = new MemoryStream(new byte[] { 65, 132, 0, 99 });
using var stream = new ArcLzwStream(input, 3, false);
var buffer = new byte[3];
foreach (var expected in new byte[] { 65, 66 })
{
var read = useAsync
? await stream.ReadAsync(buffer, 1, 1, CancellationToken.None)
: stream.Read(buffer, 1, 1);
Assert.Equal(1, read);
Assert.Equal(expected, buffer[1]);
Assert.Equal(0, buffer[0]);
}
Assert.Equal(0, stream.Read(buffer, 0, 1));
Assert.Equal(99, input.ReadByte());
}
[Fact]
public void Arc_Rejects_Negative_Compressed_Size() =>
Assert.Throws<InvalidFormatException>(() => new ArcLzwStream(Stream.Null, -1));
}

254
tests/SharpCompress.Test/Streams/BufferPoolLifetimeTests.cs
View File

@@ -6,9 +6,11 @@ using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Tar.Headers;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.LZMA.LZ;
using SharpCompress.Compressors.LZMA.Utilities;
using SharpCompress.Compressors.Shrink;
using SharpCompress.IO;
using Xunit;
@@ -16,6 +18,184 @@ namespace SharpCompress.Test.Streams;
public class BufferPoolLifetimeTests
{
[Theory]
[InlineData(false, 0, false)]
[InlineData(true, 0, false)]
[InlineData(false, 10000, false)]
[InlineData(true, 10000, false)]
[InlineData(false, 1, true)]
[InlineData(true, 1, true)]
public async Task Zip64_ExtensibleData_ReturnsBuffer(
bool useAsync,
int payloadSize,
bool truncated
)
{
var pool = new TrackingPool<byte> { ExtraCapacity = 32 };
using var input = new MemoryStream();
using (var writer = new BinaryWriter(input, System.Text.Encoding.UTF8, true))
{
writer.Write((ulong)(44 + payloadSize + (truncated ? 10000 : 0)));
writer.Write(new byte[44 + payloadSize]);
if (!truncated)
{
writer.Write((byte)99);
}
}
input.Position = 0;
var header = new Zip64DirectoryEndHeader(pool);
if (useAsync)
{
using var reader = new AsyncBinaryReader(input, leaveOpen: true);
if (truncated)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await header.Read(reader)
);
}
else
{
await header.Read(reader);
}
}
else
{
using var reader = new BinaryReader(input, System.Text.Encoding.UTF8, true);
if (truncated)
{
Assert.ThrowsAny<SharpCompressException>(() => header.Read(reader));
}
else
{
header.Read(reader);
}
}
if (!truncated)
{
Assert.Equal(99, input.ReadByte());
}
Assert.Equal(payloadSize == 0 ? 0 : 1, pool.RentCount);
Assert.Equal(pool.RentCount, pool.ReturnCount);
Assert.Empty(pool.Active);
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task Shrink_Dispose_ReturnsBuffers(bool useAsync, bool asyncDispose)
{
var pool = new TrackingPool<byte> { ExtraCapacity = 32 };
using var input = new MemoryStream(new byte[] { 65, 132, 0 });
using var stream = new ShrinkStream(input, 2, pool);
var buffer = new byte[1];
foreach (var expected in new byte[] { 65, 66 })
{
var read = useAsync
? await stream.ReadAsync(buffer, 0, 1, CancellationToken.None)
: stream.Read(buffer, 0, 1);
Assert.Equal(1, read);
Assert.Equal(expected, buffer[0]);
}
Assert.Equal(0, stream.Read(buffer, 0, 1));
Assert.Equal(2, pool.RentCount);
Assert.Equal(1, pool.ReturnCount);
Assert.Single(pool.Active);
#if !LEGACY_DOTNET
if (asyncDispose)
{
await stream.DisposeAsync();
}
else
#endif
{
stream.Dispose();
}
stream.Dispose();
Assert.Equal(2, pool.ReturnCount);
Assert.Empty(pool.Active);
Assert.True(input.CanRead);
Assert.False(stream.CanRead);
Assert.Throws<ObjectDisposedException>(() => stream.Read(buffer, 0, 1));
await Assert.ThrowsAsync<ObjectDisposedException>(() =>
stream.ReadAsync(buffer, 0, 1, CancellationToken.None)
);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Shrink_InvalidData_ReturnsBuffers(bool useAsync)
{
var pool = new TrackingPool<byte> { ExtraCapacity = 32 };
using var input = new MemoryStream(new byte[] { 65, 132, 0 });
using var stream = new ShrinkStream(input, int.MaxValue, pool);
var buffer = new byte[1];
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(() =>
stream.ReadAsync(buffer, 0, 1, CancellationToken.None)
);
}
else
{
Assert.Throws<InvalidFormatException>(() => stream.Read(buffer, 0, 1));
}
Assert.Equal(2, pool.RentCount);
Assert.Equal(2, pool.ReturnCount);
Assert.Empty(pool.Active);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Shrink_Growth_ReturnsBuffers(bool failGrowth)
{
// Literal-only nine-bit codes fill the first output rental without increasing code size.
const int count = 70000;
var input = new byte[(count * 9 + 7) / 8];
for (var i = 0; i < count; i++)
{
var bit = i * 9;
input[bit / 8] |= (byte)(65 << (bit % 8));
input[bit / 8 + 1] |= (byte)(65 >> (8 - bit % 8));
}
var pool = new TrackingPool<byte> { FailRentNumber = failGrowth ? 2 : 0 };
if (failGrowth)
{
Assert.Throws<OutOfMemoryException>(() =>
HwUnshrink.UnshrinkGrowing(input, input.Length, count, pool, out _, out _)
);
Assert.Equal(1, pool.ReturnCount);
}
else
{
var status = HwUnshrink.UnshrinkGrowing(
input,
input.Length,
count,
pool,
out var output,
out var used
);
try
{
Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status);
Assert.Equal(count, used);
Assert.All(output, value => Assert.Equal(65, value));
Assert.Equal(1, pool.ReturnCount);
}
finally
{
pool.Return(output);
}
Assert.Equal(2, pool.ReturnCount);
}
Assert.Empty(pool.Active);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
@@ -101,6 +281,74 @@ public class BufferPoolLifetimeTests
Assert.Empty(pool.Active);
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task OutWindow_LazyGrowth_PreservesPoolOwnership(bool asyncWrite, bool failGrowth)
{
var pool = new TrackingPool<byte>();
using var window = new OutWindow(pool);
window.Create(16384);
Assert.Equal(4096, window.BufferCapacity);
for (var i = 0; i < 4096; i++)
{
if (asyncWrite)
{
await window.PutByteAsync(42, CancellationToken.None);
}
else
{
window.PutByte(42);
}
}
if (failGrowth)
{
pool.FailNextRent = true;
if (asyncWrite)
{
await Assert.ThrowsAsync<OutOfMemoryException>(async () =>
await window.PutByteAsync(43, CancellationToken.None)
);
}
else
{
Assert.Throws<OutOfMemoryException>(() => window.PutByte(43));
}
Assert.Equal(0, pool.ReturnCount);
Assert.Equal(4096, window.BufferCapacity);
Assert.Equal(42, window.GetByte(0));
}
if (asyncWrite)
{
await window.PutByteAsync(43, CancellationToken.None);
}
else
{
window.PutByte(43);
}
Assert.Equal(8192, window.BufferCapacity);
Assert.Equal(42, window.GetByte(4096));
Assert.Equal(43, window.GetByte(0));
Assert.Equal(1, pool.ReturnCount);
Assert.Single(pool.Active);
if (asyncWrite)
{
await window.DisposeAsync();
}
else
{
window.Dispose();
}
Assert.Equal(2, pool.ReturnCount);
Assert.Empty(pool.Active);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
@@ -302,18 +550,20 @@ public class BufferPoolLifetimeTests
internal int RentCount { get; private set; }
internal int ReturnCount { get; private set; }
internal bool FailNextRent { get; set; }
internal int FailRentNumber { get; set; }
internal int ExtraCapacity { get; set; }
public override T[] Rent(int minimumLength)
{
RentCount++;
if (FailNextRent)
if (FailNextRent || RentCount == FailRentNumber)
{
FailNextRent = false;
#pragma warning disable CA2201 // Simulate allocation failure without exhausting process memory.
throw new OutOfMemoryException("Injected pool allocation failure.");
#pragma warning restore CA2201
}
var buffer = new T[minimumLength];
var buffer = new T[minimumLength + ExtraCapacity];
Assert.True(Active.Add(buffer));
return buffer;
}

156
tests/SharpCompress.Test/Streams/Lzma2TruncationTests.cs Normal file
View File

@@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.Xz;
using Xunit;
namespace SharpCompress.Test.Streams;
public class Lzma2TruncationTests
{
private const string TruncatedXzHex =
"FD377A585A0000016922DE360200210100000000372797D6010011616C7068610A616C7068610A616C7068610A";
private const string MutatedSevenZipBase64 =
"N3q8ryccAATrtqxIDgAAAAAAAABaAAAAAAAAAKyqq98BAApoZWxsbyB3b3JsZAEEBgABCQ4ABwsBAAEhIQEADAwACAoBhRFKDQAABQEZDAAAAAAAAAAAAAAAABETAHQAZQBzAHQALgB0AHgAdAAAABkAFAoBAI6gjYxMV90BFQYBACAAAAAAAA==";
public static IEnumerable<object[]> PartialHeaders()
{
// Start with a complete dictionary-reset chunk so a missing control byte
// cannot be rejected merely because the dictionary still needs resetting.
byte[] prefix = [0x01, 0x00, 0x00, 0x58];
yield return [prefix];
byte[][] headers =
[
[0x01, 0x00, 0x00],
[0x02, 0x00, 0x00],
[0xE0, 0x00, 0x00, 0x00, 0x04, 0x00],
];
foreach (var header in headers)
{
for (var length = 1; length < header.Length; length++)
{
var data = new byte[prefix.Length + length];
prefix.CopyTo(data, 0);
Array.Copy(header, 0, data, prefix.Length, length);
yield return [data];
}
}
}
[Theory]
[MemberData(nameof(PartialHeaders))]
public void Lzma2_Rejects_Partial_Header(byte[] data)
{
using var input = new MemoryStream(data);
using var decoder = LzmaStream.Create([0x00], input);
Assert.ThrowsAny<SharpCompressException>(() => decoder.CopyTo(Stream.Null));
}
[Theory]
[MemberData(nameof(PartialHeaders))]
public void Lzma2_ReadByte_Rejects_Partial_Header(byte[] data)
{
using var input = new MemoryStream(data);
using var decoder = LzmaStream.Create([0x00], input);
Assert.Equal(0x58, decoder.ReadByte());
Assert.ThrowsAny<SharpCompressException>(() => decoder.ReadByte());
}
[Theory]
[MemberData(nameof(PartialHeaders))]
public async Task Lzma2_Async_Rejects_Partial_Header(byte[] data)
{
using var input = new MemoryStream(data);
await using var decoder = await LzmaStream.CreateAsync([0x00], input, -1, -1);
var buffer = new byte[64];
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await decoder.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None)
);
#if !LEGACY_DOTNET
// Use a fresh decoder because parser state is undefined after a failed read.
using var memoryInput = new MemoryStream(data);
await using var memoryDecoder = await LzmaStream.CreateAsync([0x00], memoryInput, -1, -1);
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await memoryDecoder.ReadAsync(buffer.AsMemory(), CancellationToken.None)
);
#endif
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Xz_Rejects_Missing_Lzma2_End_Marker(bool useAsync)
{
var data = new byte[TruncatedXzHex.Length / 2];
for (var i = 0; i < data.Length; i++)
{
data[i] = Convert.ToByte(TruncatedXzHex.Substring(i * 2, 2), 16);
}
using var input = new MemoryStream(data);
await using var decoder = new XZStream(input);
if (useAsync)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(() =>
decoder.CopyToAsync(Stream.Null, 81920, CancellationToken.None)
);
}
else
{
Assert.ThrowsAny<SharpCompressException>(() => decoder.CopyTo(Stream.Null));
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZip_Rejects_Missing_Lzma2_End_Marker(bool useAsync)
{
using var input = new MemoryStream(Convert.FromBase64String(MutatedSevenZipBase64));
if (useAsync)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(input);
await foreach (var entry in archive.EntriesAsync)
{
await using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
await Assert.ThrowsAnyAsync<SharpCompressException>(() =>
stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None)
);
}
}
else
{
using var archive = ArchiveFactory.OpenArchive(input);
foreach (var entry in archive.Entries)
{
using var stream = entry.OpenEntryStream();
Assert.ThrowsAny<SharpCompressException>(() => stream.CopyTo(Stream.Null));
}
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma2_Accepts_End_Marker_After_Uncompressed_Chunks(bool useAsync)
{
byte[] data = [0x01, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x59, 0x00];
using var input = new MemoryStream(data);
using var output = new MemoryStream();
if (useAsync)
{
await using var decoder = await LzmaStream.CreateAsync([0x00], input, data.Length, 2);
await decoder.CopyToAsync(output, 1, CancellationToken.None);
}
else
{
using var decoder = LzmaStream.Create([0x00], input, data.Length, 2);
decoder.CopyTo(output, 1);
}
Assert.Equal(new byte[] { 0x58, 0x59 }, output.ToArray());
}
}

226
tests/SharpCompress.Test/Streams/LzmaDictionaryAllocationTests.cs Normal file
View File

@@ -0,0 +1,226 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.LZMA.LZ;
using Xunit;
namespace SharpCompress.Test.Streams;
public class LzmaDictionaryAllocationTests
{
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task Lzma_Raw_Huge_Dictionary_Decodes_With_Stream_And_Direct_APIs(
bool useAsync,
bool direct
)
{
var expected = new byte[20000];
new Random(123).NextBytes(expected);
using var packed = new MemoryStream();
byte[] properties;
using (
var encoder = LzmaStream.Create(new LzmaEncoderProperties(false, 4096), false, packed)
)
{
properties = (byte[])encoder.Properties.Clone();
encoder.Write(expected, 0, expected.Length);
}
// The compressed payload needs only a small dictionary, regardless of its advertised size.
properties[1] = 255;
properties[2] = 255;
properties[3] = 255;
properties[4] = 127;
using var input = new MemoryStream(packed.ToArray());
using var output = new MemoryStream();
if (direct)
{
using var decoder = new Decoder();
decoder.SetDecoderProperties(properties);
if (useAsync)
{
await decoder.CodeAsync(
input,
output,
input.Length,
expected.Length,
null!,
CancellationToken.None
);
}
else
{
decoder.Code(input, output, input.Length, expected.Length, null!);
}
}
else if (useAsync)
{
using var decoder = await LzmaStream.CreateAsync(
properties,
input,
input.Length,
expected.Length
);
await decoder.CopyToAsync(output, 137, CancellationToken.None);
}
else
{
using var decoder = LzmaStream.Create(properties, input, input.Length, expected.Length);
decoder.CopyTo(output, 137);
}
Assert.Equal(expected, output.ToArray());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma_Truncated_Uncompressed_Chunk_Does_Not_Allocate_From_Read_Size(
bool useAsync
)
{
using var window = new OutWindow();
window.Create(int.MaxValue);
window.SetLimit(int.MaxValue);
using var input = new MemoryStream(new byte[] { 1 });
if (useAsync)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await window.CopyStreamAsync(input, int.MaxValue, CancellationToken.None)
);
}
else
{
Assert.ThrowsAny<SharpCompressException>(() => window.CopyStream(input, int.MaxValue));
}
Assert.InRange(window.BufferCapacity, 1, 8192);
}
[Theory]
[InlineData(38)]
[InlineData(40)]
[InlineData(64)]
[InlineData(255)]
public void Lzma2_Rejects_Unsupported_Dictionary_Properties(int property)
{
Assert.ThrowsAny<SharpCompressException>(() =>
LzmaStream.Create(new byte[] { (byte)property }, Stream.Null)
);
}
[Fact]
public void Lzma_Huge_Dictionary_Starts_Small_And_Grows_With_History()
{
using var window = new OutWindow();
window.Create(int.MaxValue);
Assert.InRange(window.BufferCapacity, 1, 4096);
Assert.Equal(0, window.GetByte(0));
window.SetLimit(20000);
for (var i = 0; i < 20000; i++)
{
window.PutByte((byte)i);
}
Assert.InRange(window.BufferCapacity, 20000, 32768);
for (var i = 0; i < 20000; i++)
{
Assert.Equal((byte)i, window.GetByte(19999 - i));
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma_History_Survives_Growth_Wrap_And_Pending_Matches(bool useAsync)
{
using var window = new OutWindow();
window.Create(8192);
var output = new byte[8192];
window.SetLimit(8192);
for (var i = 0; i < output.Length; i++)
{
if (useAsync)
{
await window.PutByteAsync((byte)i, CancellationToken.None);
}
else
{
window.PutByte((byte)i);
}
}
Assert.Equal(output.Length, window.Read(output, 0, output.Length));
window.SetLimit(10);
if (useAsync)
{
await window.CopyBlockAsync(8191, 20, CancellationToken.None);
}
else
{
window.CopyBlock(8191, 20);
}
Assert.True(window.HasPending);
window.SetLimit(10);
if (useAsync)
{
await window.CopyPendingAsync(CancellationToken.None);
}
else
{
window.CopyPending();
}
Assert.False(window.HasPending);
Assert.Equal(20, window.Read(output, 0, output.Length));
for (var i = 0; i < 20; i++)
{
Assert.Equal((byte)i, output[i]);
}
window.Reset();
Assert.Equal(0, window.Total);
Assert.Equal(0, window.GetByte(0));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma_Preset_Dictionary_Does_Not_Allocate_Declared_Window(bool useAsync)
{
using var window = new OutWindow();
window.Create(int.MaxValue);
using var preset = new MemoryStream(new byte[] { 1, 2, 3 });
if (useAsync)
{
await window.TrainAsync(preset);
}
else
{
window.Train(preset);
}
Assert.Equal(3, window.GetByte(0));
Assert.Equal(1, window.GetByte(2));
Assert.InRange(window.BufferCapacity, 3, 4096);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma2_Tiny_Stream_With_Huge_Dictionary_Decodes(bool useAsync)
{
using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 });
using var output = new MemoryStream();
if (useAsync)
{
using var decoder = await LzmaStream.CreateAsync(new byte[] { 37 }, input, -1, -1);
await decoder.CopyToAsync(output, 81920, CancellationToken.None);
}
else
{
using var decoder = LzmaStream.Create(new byte[] { 37 }, input);
decoder.CopyTo(output);
}
Assert.Equal(new byte[] { 65 }, output.ToArray());
}
}

74
tests/SharpCompress.Test/Zip/ShrinkAllocationTests.cs Normal file
View File

@@ -0,0 +1,74 @@
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.Shrink;
using Xunit;
namespace SharpCompress.Test.Zip;
public class ShrinkAllocationTests
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Shrink_Rejects_Huge_Truncated_Output(bool useAsync)
{
using var input = new MemoryStream(new byte[] { 65, 0 });
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
{
using var stream = await ShrinkStream.CreateAsync(
input,
int.MaxValue,
CancellationToken.None
);
});
}
else
{
using var stream = new ShrinkStream(input, int.MaxValue);
Assert.Throws<InvalidFormatException>(() => stream.ReadByte());
}
}
[Fact]
public void Shrink_Grows_Only_For_Actual_Output()
{
var pool = ArrayPool<byte>.Shared;
var status = HwUnshrink.UnshrinkGrowing(
new byte[] { 65, 132, 0 },
3,
int.MaxValue,
pool,
out var output,
out var used
);
try
{
Assert.Equal(HwUnshrink.UnshrnkStatus.Ok, status);
Assert.Equal(2, used);
Assert.Equal(65, output[0]);
Assert.Equal(66, output[1]);
Assert.InRange(output.Length, 2, 131072);
}
finally
{
pool.Return(output);
}
}
[Fact]
public void Shrink_Preserves_Fixed_Buffer_API()
{
var output = new byte[2];
Assert.Equal(
HwUnshrink.UnshrnkStatus.Ok,
HwUnshrink.Unshrink(new byte[] { 65, 132, 0 }, 3, out _, output, 2, out var used)
);
Assert.Equal(2, used);
Assert.Equal(new byte[] { 65, 66 }, output);
}
}

68
tests/SharpCompress.Test/Zip/Zip64AllocationTests.cs Normal file
View File

@@ -0,0 +1,68 @@
using System;
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
using Xunit;
namespace SharpCompress.Test.Zip;
public class Zip64AllocationTests
{
[Theory]
[InlineData(0UL)]
[InlineData(43UL)]
[InlineData(ulong.MaxValue)]
public async Task Zip64_Rejects_Invalid_Record_Size(ulong size)
{
using var stream = CreateRecord(size, 0);
Assert.Throws<InvalidFormatException>(() =>
new Zip64DirectoryEndHeader().Read(new BinaryReader(stream))
);
stream.Position = 0;
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream))
);
}
[Theory]
[InlineData(44UL, 0)]
[InlineData(10044UL, 10000)]
public async Task Zip64_Skips_Extensible_Data_Exactly(ulong size, int payloadSize)
{
using var stream = CreateRecord(size, payloadSize);
new Zip64DirectoryEndHeader().Read(new BinaryReader(stream));
Assert.Equal(stream.Length, stream.Position);
stream.Position = 0;
await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream));
Assert.Equal(stream.Length, stream.Position);
}
[Theory]
[InlineData(2147483691UL)]
[InlineData(4294967340UL)]
public async Task Zip64_Rejects_Huge_Truncated_Extensible_Data(ulong size)
{
using var stream = CreateRecord(size, 1);
Assert.ThrowsAny<SharpCompressException>(() =>
new Zip64DirectoryEndHeader().Read(new BinaryReader(stream))
);
stream.Position = 0;
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await new Zip64DirectoryEndHeader().Read(new AsyncBinaryReader(stream))
);
}
private static MemoryStream CreateRecord(ulong size, int payloadSize)
{
var stream = new MemoryStream();
using (var writer = new BinaryWriter(stream, System.Text.Encoding.UTF8, true))
{
writer.Write(size);
writer.Write(new byte[44 + payloadSize]);
}
stream.Position = 0;
return stream;
}
}