mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-11-15 17:02:03 +00:00
Merge commit from fork
Fix GHSA-xmfm-xg76-qx62: bound decompression dictionary allocations
This commit is contained in:
33
docs/API.md
33
docs/API.md
@@ -327,6 +327,39 @@ await writer.WriteAllAsync("C:\\source", "*", SearchOption.AllDirectories, cance
|
||||
|
||||
## Common Options
|
||||
|
||||
### Decompression dictionary limits
|
||||
|
||||
`SharpCompress.Common.DictionarySizeLimits` contains process-wide static size limits
|
||||
in bytes. Set them once at application startup, not around individual requests:
|
||||
|
||||
```csharp
|
||||
// Opt in only when trusted archives require larger dictionaries.
|
||||
DictionarySizeLimits.Rar5 = 256L << 20;
|
||||
```
|
||||
|
||||
| Static property | Default maximum |
|
||||
| --- | --- |
|
||||
| `Rar4` | 4 MiB (the RAR4 format maximum) |
|
||||
| `Rar5` | 64 MiB |
|
||||
| `Lzma` (raw LZMA, LZip, ZIP, 7z) | 64 MiB |
|
||||
| `Lzma2` (XZ, 7z) | 64 MiB |
|
||||
| `Ppmd` (ZIP, 7z, RAR models) | 64 MiB |
|
||||
| `ZStandard` (streaming windows) | 128 MiB |
|
||||
|
||||
Limits are read directly by the shared codecs before dictionary/model allocation,
|
||||
so Archive, Reader, direct decoder, sync, and async paths use the same settings.
|
||||
No reader options or decoder overloads are needed. Settings require positive values;
|
||||
invalid assignments throw `ArgumentOutOfRangeException` without changing the setting.
|
||||
Oversized dictionaries throw `InvalidFormatException`; Zstandard reports
|
||||
`ZstdException` with `ZSTD_error_frameParameter_windowTooLarge`. Zstandard's minimum
|
||||
configurable window limit is 1 KiB. Format and engine restrictions still apply.
|
||||
|
||||
These are dictionary/window or model limits, not total-memory, output-size, or
|
||||
concurrency limits. Pool rounding and allocator overhead can increase actual memory.
|
||||
Shared PPMd allocators also apply the model limit during compression. Fixed/bounded
|
||||
codecs retain their format-defined bounds. Custom providers must enforce their own
|
||||
limits. Zstandard's one-shot `Unwrap` APIs retain their existing output-size bounds.
|
||||
|
||||
### ReaderOptions
|
||||
|
||||
Use preset properties and fluent helpers for common configurations:
|
||||
|
||||
@@ -483,3 +483,15 @@ ZIP Shrink output buffering grows with decoded output rather than the declared u
|
||||
7z header counts and coder property lengths are validated against their backing metadata before allocation. File counts are checked against parsed substreams and empty-stream bitmaps, including external metadata. Encoded headers and additional metadata are buffered incrementally as the decoder produces bytes, rather than preallocating the declared unpacked size.
|
||||
|
||||
LZMA/LZMA2 dictionary storage starts small and grows with decoded history, up to the supported dictionary size. These safeguards avoid allocations based solely on forged sizes; they do not impose an overall decompressed-output or process-memory limit.
|
||||
|
||||
Shared codecs also check `SharpCompress.Common.DictionarySizeLimits` before
|
||||
allocating dictionaries/windows or PPMd models. Defaults are 4 MiB for RAR4,
|
||||
64 MiB for RAR5, LZMA (including LZip), LZMA2 (including XZ), and PPMd, and
|
||||
128 MiB for Zstandard streaming windows. A tiny RAR5 entry therefore cannot rent
|
||||
a 256 MiB or 1 GiB window under the default settings.
|
||||
|
||||
These settings are **process-wide**, not per-reader options. Configure them once
|
||||
at startup if trusted input requires larger dictionaries, for example
|
||||
`DictionarySizeLimits.Rar5 = 256L << 20;`. Do not temporarily raise them around
|
||||
untrusted requests. Bound extraction output and concurrency separately; dictionary
|
||||
limits are not a total-memory budget. See the [API reference](API.md#decompression-dictionary-limits).
|
||||
|
||||
80
src/SharpCompress/Common/DictionarySizeLimits.cs
Normal file
80
src/SharpCompress/Common/DictionarySizeLimits.cs
Normal file
@@ -0,0 +1,80 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace SharpCompress.Common;
|
||||
|
||||
/// <summary>
|
||||
/// Process-wide dictionary/window and PPMd model size limits, in bytes.
|
||||
/// Configure once at application startup, not for individual extraction requests.
|
||||
/// These limits do not bound total memory, allocator overhead, or decompressed output.
|
||||
/// </summary>
|
||||
public static class DictionarySizeLimits
|
||||
{
|
||||
private static long rar4 = 4L << 20;
|
||||
private static long rar5 = 64L << 20;
|
||||
private static long lzma = 64L << 20;
|
||||
private static long lzma2 = 64L << 20;
|
||||
private static long ppmd = 64L << 20;
|
||||
private static long zStandard = 128L << 20;
|
||||
|
||||
/// <summary>RAR4 window limit. Defaults to the format maximum of 4 MiB.</summary>
|
||||
public static long Rar4
|
||||
{
|
||||
get => Interlocked.Read(ref rar4);
|
||||
set => SetMaximum(ref rar4, value, nameof(Rar4));
|
||||
}
|
||||
|
||||
/// <summary>RAR5 window limit. Defaults to 64 MiB.</summary>
|
||||
public static long Rar5
|
||||
{
|
||||
get => Interlocked.Read(ref rar5);
|
||||
set => SetMaximum(ref rar5, value, nameof(Rar5));
|
||||
}
|
||||
|
||||
/// <summary>LZMA dictionary limit, including LZip, ZIP, and 7z. Defaults to 64 MiB.</summary>
|
||||
public static long Lzma
|
||||
{
|
||||
get => Interlocked.Read(ref lzma);
|
||||
set => SetMaximum(ref lzma, value, nameof(Lzma));
|
||||
}
|
||||
|
||||
/// <summary>LZMA2 dictionary limit, including XZ and 7z. Defaults to 64 MiB.</summary>
|
||||
public static long Lzma2
|
||||
{
|
||||
get => Interlocked.Read(ref lzma2);
|
||||
set => SetMaximum(ref lzma2, value, nameof(Lzma2));
|
||||
}
|
||||
|
||||
/// <summary>PPMd model limit, including ZIP, 7z, and RAR models. Defaults to 64 MiB.</summary>
|
||||
public static long Ppmd
|
||||
{
|
||||
get => Interlocked.Read(ref ppmd);
|
||||
set => SetMaximum(ref ppmd, value, nameof(Ppmd));
|
||||
}
|
||||
|
||||
/// <summary>Zstandard streaming window limit. Defaults to 128 MiB.</summary>
|
||||
public static long ZStandard
|
||||
{
|
||||
get => Interlocked.Read(ref zStandard);
|
||||
set => SetMaximum(ref zStandard, value, nameof(ZStandard));
|
||||
}
|
||||
|
||||
private static void SetMaximum(ref long field, long value, string name)
|
||||
{
|
||||
if (value <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(name, "Dictionary size limits must be positive.");
|
||||
}
|
||||
Interlocked.Exchange(ref field, value);
|
||||
}
|
||||
|
||||
internal static void Validate(long size, long maximum, string format)
|
||||
{
|
||||
if (size <= 0 || size > maximum)
|
||||
{
|
||||
throw new InvalidFormatException(
|
||||
$"{format} dictionary size {size} is invalid or exceeds the maximum of {maximum} bytes."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.LZMA.LZ;
|
||||
using SharpCompress.Compressors.LZMA.RangeCoder;
|
||||
|
||||
@@ -229,6 +230,7 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
|
||||
}
|
||||
_outWindow = new OutWindow();
|
||||
var blockSize = Math.Max(_dictionarySize, (1 << 12));
|
||||
DictionarySizeLimits.Validate(blockSize, DictionarySizeLimits.Lzma, "LZMA");
|
||||
_outWindow.Create(blockSize);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Buffers.Binary;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.LZMA.LZ;
|
||||
using SharpCompress.IO;
|
||||
|
||||
@@ -54,6 +55,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
|
||||
if (!isLzma2)
|
||||
{
|
||||
_dictionarySize = BinaryPrimitives.ReadInt32LittleEndian(properties.AsSpan(1));
|
||||
DictionarySizeLimits.Validate(_dictionarySize, DictionarySizeLimits.Lzma, "LZMA");
|
||||
_decoder = new Decoder();
|
||||
_decoder.SetDecoderProperties(properties);
|
||||
Properties = properties;
|
||||
@@ -72,6 +74,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
|
||||
}
|
||||
_dictionarySize = 2 | (properties[0] & 1);
|
||||
_dictionarySize <<= (properties[0] >> 1) + 11;
|
||||
DictionarySizeLimits.Validate(_dictionarySize, DictionarySizeLimits.Lzma2, "LZMA2");
|
||||
|
||||
Properties = new byte[1];
|
||||
_availableBytes = 0;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Text;
|
||||
using SharpCompress.Common;
|
||||
|
||||
namespace SharpCompress.Compressors.PPMd.H;
|
||||
|
||||
@@ -155,6 +156,7 @@ internal class SubAllocator : IDisposable
|
||||
|
||||
public virtual bool StartSubAllocator(int saSize)
|
||||
{
|
||||
DictionarySizeLimits.Validate(saSize, DictionarySizeLimits.Ppmd, "PPMd");
|
||||
var t = saSize;
|
||||
if (_subAllocatorSize == t)
|
||||
{
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#nullable disable
|
||||
|
||||
using SharpCompress.Common;
|
||||
|
||||
namespace SharpCompress.Compressors.PPMd.I1;
|
||||
|
||||
/// Allocate a single, large array and then provide sections of this array to callers. Callers are provided with
|
||||
@@ -128,6 +130,9 @@ internal class Allocator
|
||||
/// <param name="allocatorSize"></param>
|
||||
public void Start(int allocatorSize)
|
||||
{
|
||||
// PpmdProperties starts this allocator while parsing ZIP properties, so
|
||||
// enforce the limit here, before any attacker-selected allocation.
|
||||
DictionarySizeLimits.Validate(allocatorSize, DictionarySizeLimits.Ppmd, "PPMd");
|
||||
var size = (uint)allocatorSize;
|
||||
if (_allocatorSize != size)
|
||||
{
|
||||
|
||||
@@ -130,6 +130,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
|
||||
|
||||
private void Init()
|
||||
{
|
||||
DictionarySizeLimits.Validate(PackDef.MAXWINSIZE, DictionarySizeLimits.Rar4, "RAR4");
|
||||
if (this.window is null)
|
||||
{
|
||||
this.window = ArrayPool<byte>.Shared.Rent(PackDef.MAXWINSIZE);
|
||||
|
||||
@@ -99,14 +99,22 @@ internal sealed partial class Unpack : BitInput
|
||||
WinSize = MinAllocSize;
|
||||
}
|
||||
|
||||
// Validate before renting or reusing a solid window. The RAR5 header can
|
||||
// request a huge dictionary even when there are no compressed bytes.
|
||||
DictionarySizeLimits.Validate(
|
||||
WinSize,
|
||||
fileHeader.IsRar5 ? DictionarySizeLimits.Rar5 : DictionarySizeLimits.Rar4,
|
||||
fileHeader.IsRar5 ? "RAR5" : "RAR4"
|
||||
);
|
||||
|
||||
if (WinSize <= MaxWinSize) // Use the already allocated window.
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if ((WinSize >> 16) > 0x10000) // Window size must not exceed 4 GB.
|
||||
if (WinSize > int.MaxValue)
|
||||
{
|
||||
return;
|
||||
throw new InvalidFormatException("RAR window exceeds the supported array size.");
|
||||
}
|
||||
|
||||
// Archiving code guarantees that window size does not grow in the same
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.ZStandard.Unsafe;
|
||||
|
||||
namespace SharpCompress.Compressors.ZStandard;
|
||||
@@ -10,6 +11,21 @@ public unsafe class Decompressor : IDisposable
|
||||
public Decompressor()
|
||||
{
|
||||
handle = SafeDctxHandle.Create();
|
||||
try
|
||||
{
|
||||
using var dctx = handle.Acquire();
|
||||
Unsafe
|
||||
.Methods.ZSTD_DCtx_setMaxWindowSize(
|
||||
dctx,
|
||||
checked((nuint)DictionarySizeLimits.ZStandard)
|
||||
)
|
||||
.EnsureZstdSuccess();
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public void SetParameter(ZSTD_dParameter parameter, int value)
|
||||
|
||||
@@ -3201,6 +3201,17 @@ public static unsafe partial class Methods
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the limit before the single-pass shortcut too, so it
|
||||
// does not depend on the caller's output buffer size.
|
||||
zds->fParams.windowSize =
|
||||
zds->fParams.windowSize > 1U << 10 ? zds->fParams.windowSize : 1U << 10;
|
||||
if (zds->fParams.windowSize > zds->maxWindowSize)
|
||||
{
|
||||
return unchecked(
|
||||
(nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge)
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
zds->fParams.frameContentSize != unchecked(0UL - 1)
|
||||
&& zds->fParams.frameType != ZSTD_frameType_e.ZSTD_skippableFrame
|
||||
@@ -3282,15 +3293,6 @@ public static unsafe partial class Methods
|
||||
zds->stage = ZSTD_dStage.ZSTDds_decodeBlockHeader;
|
||||
}
|
||||
|
||||
zds->fParams.windowSize =
|
||||
zds->fParams.windowSize > 1U << 10 ? zds->fParams.windowSize : 1U << 10;
|
||||
if (zds->fParams.windowSize > zds->maxWindowSize)
|
||||
{
|
||||
return unchecked(
|
||||
(nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge)
|
||||
);
|
||||
}
|
||||
|
||||
if (zds->maxBlockSizeParam != 0)
|
||||
{
|
||||
zds->fParams.blockSizeMax =
|
||||
|
||||
167
tests/SharpCompress.Test/Rar/RarDictionarySizeLimitTests.cs
Normal file
167
tests/SharpCompress.Test/Rar/RarDictionarySizeLimitTests.cs
Normal file
@@ -0,0 +1,167 @@
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Xz;
|
||||
using SharpCompress.Readers;
|
||||
using SharpCompress.Test.Streams;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Rar;
|
||||
|
||||
[Collection(DictionarySizeLimitCollection.Name)]
|
||||
public class RarDictionarySizeLimitTests
|
||||
{
|
||||
// The advisory's complete 72-byte archives have valid header CRCs, no packed
|
||||
// data, and just 45,056 bytes of declared output. Neither needs a large window.
|
||||
private const string Rar256MiB =
|
||||
"526172211a070100b28180510c01050800070101c496868000c63c6d612a02030b8080000480e00220c809b1cf8059000c6578652f746573742e6578650a03020058a2ce32ffc101";
|
||||
private const string Rar1GiB =
|
||||
"526172211a070100b28180510c01050800070101c4968680000fd7ea442a02030b8080000480e00220c809b1cf8069000c6578652f746573742e6578650a03020058a2ce32ffc101";
|
||||
|
||||
public static IEnumerable<object[]> Payloads()
|
||||
{
|
||||
foreach (var hex in new[] { Rar256MiB, Rar1GiB })
|
||||
{
|
||||
foreach (var archiveApi in new[] { false, true })
|
||||
{
|
||||
foreach (var useAsync in new[] { false, true })
|
||||
{
|
||||
yield return new object[] { hex, archiveApi, useAsync };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Payloads))]
|
||||
public async Task Rar5_Huge_Window_Is_Rejected_Before_Decoding(
|
||||
string hex,
|
||||
bool archiveApi,
|
||||
bool useAsync
|
||||
)
|
||||
{
|
||||
var bytes = DecodeHex(hex);
|
||||
Assert.Equal(72, bytes.Length);
|
||||
using var input = new MemoryStream(bytes);
|
||||
var options = ReaderOptions.ForExternalStream;
|
||||
InvalidFormatException exception;
|
||||
if (archiveApi && useAsync)
|
||||
{
|
||||
await using var archive = await ArchiveFactory.OpenAsyncArchive(
|
||||
input,
|
||||
options,
|
||||
CancellationToken.None
|
||||
);
|
||||
var entry = await archive.EntriesAsync.FirstAsync();
|
||||
Assert.Equal(45056, entry.Size);
|
||||
Assert.Equal(0, entry.CompressedSize);
|
||||
exception = await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
{
|
||||
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
|
||||
});
|
||||
}
|
||||
else if (archiveApi)
|
||||
{
|
||||
using var archive = ArchiveFactory.OpenArchive(input, options);
|
||||
var entry = archive.Entries.First();
|
||||
Assert.Equal(45056, entry.Size);
|
||||
Assert.Equal(0, entry.CompressedSize);
|
||||
exception = Assert.Throws<InvalidFormatException>(() =>
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
});
|
||||
}
|
||||
else if (useAsync)
|
||||
{
|
||||
using var forwardOnly = new NonSeekableStream(input);
|
||||
await using var reader = await ReaderFactory.OpenAsyncReader(
|
||||
forwardOnly,
|
||||
options,
|
||||
CancellationToken.None
|
||||
);
|
||||
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
|
||||
exception = await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
using var forwardOnly = new NonSeekableStream(input);
|
||||
using var reader = ReaderFactory.OpenReader(forwardOnly, options);
|
||||
Assert.True(reader.MoveToNextEntry());
|
||||
exception = Assert.Throws<InvalidFormatException>(() =>
|
||||
reader.WriteEntryTo(Stream.Null)
|
||||
);
|
||||
}
|
||||
Assert.Contains("RAR5 dictionary size", exception.Message);
|
||||
Assert.Contains(DictionarySizeLimits.Rar5.ToString(), exception.Message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Rar5_Custom_Limit_Accounts_For_Minimum_Window_Allocation(bool useAsync)
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
var bytes = DecodeHex(Rar256MiB);
|
||||
// Change the compression-info vint from 0x2c80 to 0x0080: a 128 KiB
|
||||
// declared dictionary. The unpacker requires at least a 256 KiB window.
|
||||
bytes[46] = 1;
|
||||
var crc = ~Crc32.Update(Crc32.DefaultSeed, bytes.AsSpan(29));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(25), crc);
|
||||
foreach (var maximum in new[] { 128L << 10, 256L << 10 })
|
||||
{
|
||||
using var input = new MemoryStream(bytes);
|
||||
DictionarySizeLimits.Rar5 = maximum;
|
||||
var options = ReaderOptions.ForExternalStream;
|
||||
if (useAsync)
|
||||
{
|
||||
await using var archive = await ArchiveFactory.OpenAsyncArchive(
|
||||
input,
|
||||
options,
|
||||
CancellationToken.None
|
||||
);
|
||||
var entry = await archive.EntriesAsync.FirstAsync();
|
||||
if (maximum < 256L << 10)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
{
|
||||
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
using var archive = ArchiveFactory.OpenArchive(input, options);
|
||||
var entry = archive.Entries.First();
|
||||
if (maximum < 256L << 10)
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() =>
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] DecodeHex(string hex) =>
|
||||
Enumerable
|
||||
.Range(0, hex.Length / 2)
|
||||
.Select(i => Convert.ToByte(hex.Substring(i * 2, 2), 16))
|
||||
.ToArray();
|
||||
}
|
||||
389
tests/SharpCompress.Test/Streams/DictionarySizeLimitTests.cs
Normal file
389
tests/SharpCompress.Test/Streams/DictionarySizeLimitTests.cs
Normal file
@@ -0,0 +1,389 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors;
|
||||
using SharpCompress.Compressors.LZMA;
|
||||
using SharpCompress.Compressors.PPMd;
|
||||
using SharpCompress.Compressors.PPMd.H;
|
||||
using SharpCompress.Compressors.ZStandard;
|
||||
using SharpCompress.Compressors.ZStandard.Unsafe;
|
||||
using SharpCompress.Readers;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Streams;
|
||||
|
||||
// Static settings must not change while other collections are extracting archives.
|
||||
[CollectionDefinition(Name, DisableParallelization = true)]
|
||||
public class DictionarySizeLimitCollection
|
||||
{
|
||||
public const string Name = "Dictionary size limits";
|
||||
}
|
||||
|
||||
internal sealed class DictionarySizeLimitScope : IDisposable
|
||||
{
|
||||
private readonly long rar4 = DictionarySizeLimits.Rar4;
|
||||
private readonly long rar5 = DictionarySizeLimits.Rar5;
|
||||
private readonly long lzma = DictionarySizeLimits.Lzma;
|
||||
private readonly long lzma2 = DictionarySizeLimits.Lzma2;
|
||||
private readonly long ppmd = DictionarySizeLimits.Ppmd;
|
||||
private readonly long zStandard = DictionarySizeLimits.ZStandard;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
DictionarySizeLimits.Rar4 = rar4;
|
||||
DictionarySizeLimits.Rar5 = rar5;
|
||||
DictionarySizeLimits.Lzma = lzma;
|
||||
DictionarySizeLimits.Lzma2 = lzma2;
|
||||
DictionarySizeLimits.Ppmd = ppmd;
|
||||
DictionarySizeLimits.ZStandard = zStandard;
|
||||
}
|
||||
}
|
||||
|
||||
[Collection(DictionarySizeLimitCollection.Name)]
|
||||
public class DictionarySizeLimitTests : TestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Static_Limits_Have_Format_Defaults_And_Reject_Nonpositive_Values()
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
Assert.Equal(4L << 20, DictionarySizeLimits.Rar4);
|
||||
Assert.Equal(64L << 20, DictionarySizeLimits.Rar5);
|
||||
Assert.Equal(64L << 20, DictionarySizeLimits.Lzma);
|
||||
Assert.Equal(64L << 20, DictionarySizeLimits.Lzma2);
|
||||
Assert.Equal(64L << 20, DictionarySizeLimits.Ppmd);
|
||||
Assert.Equal(128L << 20, DictionarySizeLimits.ZStandard);
|
||||
DictionarySizeLimits.Rar5 = 256L << 20;
|
||||
Assert.Equal(256L << 20, DictionarySizeLimits.Rar5);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Rar4 = 0);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Rar5 = -1);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Lzma = 0);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Lzma2 = -1);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Ppmd = 0);
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.ZStandard = -1);
|
||||
Assert.Equal(256L << 20, DictionarySizeLimits.Rar5);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Lzma_And_Lzma2_Reject_Excessive_Declared_Dictionaries(bool useAsync)
|
||||
{
|
||||
foreach (var properties in new[] { new byte[] { 0x5d, 0, 0, 0, 0x10 }, new byte[] { 30 } })
|
||||
{
|
||||
using var input = new MemoryStream();
|
||||
if (useAsync)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await LzmaStream.CreateAsync(properties, input, -1, -1)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() => LzmaStream.Create(properties, input));
|
||||
}
|
||||
Assert.True(input.CanRead);
|
||||
Assert.Equal(0, input.Position);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Direct_Lzma_Decoder_Enforces_Default_Limit(bool useAsync)
|
||||
{
|
||||
using var decoder = new Decoder();
|
||||
decoder.SetDecoderProperties(new byte[] { 0x5d, 0, 0, 0, 0x10 });
|
||||
if (useAsync)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await decoder.CodeAsync(
|
||||
Stream.Null,
|
||||
Stream.Null,
|
||||
0,
|
||||
1,
|
||||
null!,
|
||||
CancellationToken.None
|
||||
)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() =>
|
||||
decoder.Code(Stream.Null, Stream.Null, 0, 1, null!)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Lzma2_Static_Limit_Is_Inclusive(bool useAsync)
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 });
|
||||
DictionarySizeLimits.Lzma2 = 4095;
|
||||
if (useAsync)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await LzmaStream.CreateAsync(new byte[] { 0 }, input, -1, -1)
|
||||
);
|
||||
DictionarySizeLimits.Lzma2 = 4096;
|
||||
using var decoder = await LzmaStream.CreateAsync(new byte[] { 0 }, input, -1, -1);
|
||||
var buffer = new byte[1];
|
||||
Assert.Equal(1, await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None));
|
||||
Assert.Equal(65, buffer[0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() => LzmaStream.Create(new byte[] { 0 }, input));
|
||||
DictionarySizeLimits.Lzma2 = 4096;
|
||||
using var decoder = LzmaStream.Create(new byte[] { 0 }, input);
|
||||
Assert.Equal(65, decoder.ReadByte());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ppmd_Properties_Reject_Large_Models_Before_Eager_Allocation()
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() => new PpmdProperties(new byte[] { 0xf5, 0x0f }));
|
||||
Assert.Throws<InvalidFormatException>(() => new PpmdProperties(256 << 20, 6));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task SevenZip_Ppmd_Rejects_Large_Models_Before_Allocation(bool useAsync)
|
||||
{
|
||||
var properties = new PpmdProperties(new byte[] { 6, 0, 0, 0, 0x10 });
|
||||
using var input = new MemoryStream();
|
||||
if (useAsync)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await PpmdStream.CreateAsync(properties, input, false, CancellationToken.None)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() =>
|
||||
PpmdStream.Create(properties, input, false)
|
||||
);
|
||||
}
|
||||
Assert.True(input.CanRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Rar_Ppmd_Suballocator_Enforces_Limit_Before_Renting()
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
DictionarySizeLimits.Ppmd = 1 << 20;
|
||||
using var allocator = new SubAllocator();
|
||||
Assert.Throws<InvalidFormatException>(() => allocator.StartSubAllocator(2 << 20));
|
||||
Assert.Equal(0, allocator.GetAllocatedMemory());
|
||||
Assert.True(allocator.StartSubAllocator(1 << 20));
|
||||
Assert.Equal(1 << 20, allocator.GetAllocatedMemory());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task LZip_Uses_Static_Lzma_Limit(bool useAsync)
|
||||
{
|
||||
var bytes = new byte[26];
|
||||
new byte[] { (byte)'L', (byte)'Z', (byte)'I', (byte)'P', 1, 28 }.CopyTo(bytes, 0);
|
||||
using var input = new MemoryStream(bytes);
|
||||
if (useAsync)
|
||||
{
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await LZipStream.CreateAsync(
|
||||
input,
|
||||
CompressionMode.Decompress,
|
||||
cancellationToken: CancellationToken.None
|
||||
)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Throws<InvalidFormatException>(() =>
|
||||
LZipStream.Create(input, CompressionMode.Decompress)
|
||||
);
|
||||
}
|
||||
Assert.True(input.CanRead);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Zstandard_Enforces_Default_And_Exact_Static_Window_Limits(bool useAsync)
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
var defaultMaximum = DictionarySizeLimits.ZStandard;
|
||||
foreach (var maximum in new long?[] { null, 16383, 16384 })
|
||||
{
|
||||
DictionarySizeLimits.ZStandard = maximum ?? defaultMaximum;
|
||||
// A 1 GiB or 16 KiB window with a final empty raw block.
|
||||
using var input = new MemoryStream(
|
||||
new byte[]
|
||||
{
|
||||
0x28,
|
||||
0xb5,
|
||||
0x2f,
|
||||
0xfd,
|
||||
0,
|
||||
maximum is null ? (byte)0xa0 : (byte)0x20,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
}
|
||||
);
|
||||
using var decoder = new DecompressionStream(input);
|
||||
var buffer = new byte[1];
|
||||
if (maximum == 16384)
|
||||
{
|
||||
var read = useAsync
|
||||
? await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None)
|
||||
: decoder.Read(buffer, 0, 1);
|
||||
Assert.Equal(0, read);
|
||||
}
|
||||
else
|
||||
{
|
||||
var exception = useAsync
|
||||
? await Assert.ThrowsAsync<ZstdException>(async () =>
|
||||
await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None)
|
||||
)
|
||||
: Assert.Throws<ZstdException>(() => decoder.Read(buffer, 0, 1));
|
||||
Assert.Equal(
|
||||
ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge,
|
||||
exception.Code
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<object[]> ArchiveCases()
|
||||
{
|
||||
foreach (
|
||||
var file in new[]
|
||||
{
|
||||
"Rar.rar",
|
||||
"Rar4.rar",
|
||||
"Rar5.rar",
|
||||
"Rar5.solid.rar",
|
||||
"Zip.lzma.zip",
|
||||
"Zip.ppmd.zip",
|
||||
"WinZip27_XZ.zipx",
|
||||
"WinZip27_ZSTD.zipx",
|
||||
"7Zip.LZMA.7z",
|
||||
"7Zip.LZMA2.7z",
|
||||
"7Zip.PPMd.7z",
|
||||
"7Zip.ZSTD.7z",
|
||||
"Tar.tar.xz",
|
||||
"Tar.tar.lz",
|
||||
"Tar.tar.zst",
|
||||
}
|
||||
)
|
||||
{
|
||||
foreach (var archiveApi in new[] { false, true })
|
||||
{
|
||||
foreach (var useAsync in new[] { false, true })
|
||||
{
|
||||
if (
|
||||
(!archiveApi && file.EndsWith(".7z", StringComparison.Ordinal))
|
||||
|| (archiveApi && file.StartsWith("Tar.", StringComparison.Ordinal))
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
yield return new object[] { file, archiveApi, useAsync };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(ArchiveCases))]
|
||||
public async Task Static_Limits_Apply_To_Archive_And_Reader_Paths(
|
||||
string file,
|
||||
bool archiveApi,
|
||||
bool useAsync
|
||||
)
|
||||
{
|
||||
using (var limits = new DictionarySizeLimitScope())
|
||||
{
|
||||
DictionarySizeLimits.Rar4 =
|
||||
DictionarySizeLimits.Rar5 =
|
||||
DictionarySizeLimits.Lzma =
|
||||
DictionarySizeLimits.Lzma2 =
|
||||
DictionarySizeLimits.Ppmd =
|
||||
DictionarySizeLimits.ZStandard =
|
||||
1024;
|
||||
await Assert.ThrowsAnyAsync<SharpCompressException>(() =>
|
||||
Extract(file, archiveApi, useAsync)
|
||||
);
|
||||
}
|
||||
await Extract(file, archiveApi, useAsync);
|
||||
}
|
||||
|
||||
private static async Task Extract(string file, bool archiveApi, bool useAsync)
|
||||
{
|
||||
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, file));
|
||||
var options = ReaderOptions.ForExternalStream;
|
||||
if (archiveApi && useAsync)
|
||||
{
|
||||
await using var archive = await ArchiveFactory.OpenAsyncArchive(
|
||||
input,
|
||||
options,
|
||||
CancellationToken.None
|
||||
);
|
||||
await foreach (var entry in archive.EntriesAsync)
|
||||
{
|
||||
if (!entry.IsDirectory)
|
||||
{
|
||||
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
|
||||
await stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (archiveApi)
|
||||
{
|
||||
using var archive = ArchiveFactory.OpenArchive(input, options);
|
||||
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
stream.CopyTo(Stream.Null);
|
||||
}
|
||||
}
|
||||
else if (useAsync)
|
||||
{
|
||||
using var forwardOnly = new NonSeekableStream(input);
|
||||
await using var reader = await ReaderFactory.OpenAsyncReader(
|
||||
forwardOnly,
|
||||
options,
|
||||
CancellationToken.None
|
||||
);
|
||||
while (await reader.MoveToNextEntryAsync(CancellationToken.None))
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
using var forwardOnly = new NonSeekableStream(input);
|
||||
using var reader = ReaderFactory.OpenReader(forwardOnly, options);
|
||||
while (reader.MoveToNextEntry())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
reader.WriteEntryTo(Stream.Null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Streams;
|
||||
|
||||
[Collection(DictionarySizeLimitCollection.Name)]
|
||||
public class LzmaDictionaryAllocationTests
|
||||
{
|
||||
[Theory]
|
||||
@@ -22,6 +23,8 @@ public class LzmaDictionaryAllocationTests
|
||||
bool direct
|
||||
)
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
DictionarySizeLimits.Lzma = int.MaxValue;
|
||||
var expected = new byte[20000];
|
||||
new Random(123).NextBytes(expected);
|
||||
using var packed = new MemoryStream();
|
||||
@@ -210,6 +213,8 @@ public class LzmaDictionaryAllocationTests
|
||||
[InlineData(true)]
|
||||
public async Task Lzma2_Tiny_Stream_With_Huge_Dictionary_Decodes(bool useAsync)
|
||||
{
|
||||
using var limits = new DictionarySizeLimitScope();
|
||||
DictionarySizeLimits.Lzma2 = int.MaxValue;
|
||||
using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 });
|
||||
using var output = new MemoryStream();
|
||||
if (useAsync)
|
||||
|
||||
Reference in New Issue
Block a user