Merge pull request #1492 from adamhathcock/adam/merge-release-into-master

Merge release into master
This commit is contained in:
Adam Hathcock
2026-11-14 09:37:37 +00:00
committed by GitHub
57 changed files with 4302 additions and 232 deletions

View File

@@ -15,6 +15,7 @@ on:
permissions:
contents: read
id-token: write
jobs:
build-and-publish:
@@ -60,9 +61,16 @@ jobs:
name: ${{ matrix.os }}-nuget-package
path: artifacts/*.nupkg
- name: NuGet login
if: success() && matrix.os == 'windows-latest' && startsWith(github.ref, 'refs/tags/')
uses: NuGet/login@v1
id: nuget-login
with:
user: adamhathcock
# Push to NuGet.org only for version tag pushes (Windows only)
- name: Push to NuGet
if: success() && matrix.os == 'windows-latest' && startsWith(github.ref, 'refs/tags/')
run: dotnet run --project build/build.csproj -- push-to-nuget
env:
NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}
NUGET_API_KEY: ${{ steps.nuget-login.outputs.NUGET_API_KEY }}

View File

@@ -368,6 +368,41 @@ await writer.WriteAllAsync("C:\\source", "*", SearchOption.AllDirectories, cance
## Common Options
### Decompression dictionary limits
`SharpCompress.Common.DictionarySizeLimits` contains process-wide static size limits
in bytes. Set them once at application startup, not around individual requests:
```csharp
// Opt in only when trusted archives require larger dictionaries.
DictionarySizeLimits.Rar5 = 256L << 20;
```
| Static property | Default maximum |
| --- | --- |
| `Rar4` | 4 MiB (the RAR4 format maximum) |
| `Rar5` | 64 MiB |
| `Lzma` (raw LZMA, LZip, ZIP, 7z) | 64 MiB |
| `Lzma2` (XZ, 7z) | 64 MiB |
| `Ppmd` (ZIP, 7z, RAR models) | 64 MiB |
| `ZStandard` (streaming windows) | 128 MiB |
Limits are read directly by the shared codecs before dictionary/model allocation,
so Archive, Reader, direct decoder, sync, and async paths use the same settings.
No reader options or decoder overloads are needed. Settings require positive values;
invalid assignments throw `ArgumentOutOfRangeException` without changing the setting.
Oversized dictionaries throw `InvalidFormatException`; Zstandard reports
`ZstdException` with `ZSTD_error_frameParameter_windowTooLarge`. Zstandard's minimum
configurable window limit is 1 KiB, and its maximum is 1 GiB in a 32-bit process
or 2 GiB in a 64-bit process. Values outside that inclusive range are rejected
at assignment. Format and engine restrictions still apply.
These are dictionary/window or model limits, not total-memory, output-size, or
concurrency limits. Pool rounding and allocator overhead can increase actual memory.
Shared PPMd allocators also apply the model limit during compression. Fixed/bounded
codecs retain their format-defined bounds. Custom providers must enforce their own
limits. Zstandard's one-shot `Unwrap` APIs retain their existing output-size bounds.
### ReaderOptions
Use preset properties and fluent helpers for common configurations:

View File

@@ -525,4 +525,18 @@ ARC LZW compressed data is buffered incrementally from the bytes actually presen
ZIP Shrink output buffering grows with decoded output rather than the declared uncompressed size. Invalid codes and output-size mismatches are rejected.
7z header counts and coder property lengths are validated against their backing metadata before allocation. File counts are checked against parsed substreams and empty-stream bitmaps, including external metadata. Encoded headers and additional metadata are buffered incrementally as the decoder produces bytes, rather than preallocating the declared unpacked size.
LZMA/LZMA2 dictionary storage starts small and grows with decoded history, up to the supported dictionary size. These safeguards avoid allocations based solely on forged sizes; they do not impose an overall decompressed-output or process-memory limit.
Shared codecs also check `SharpCompress.Common.DictionarySizeLimits` before
allocating dictionaries/windows or PPMd models. Defaults are 4 MiB for RAR4,
64 MiB for RAR5, LZMA (including LZip), LZMA2 (including XZ), and PPMd, and
128 MiB for Zstandard streaming windows. A tiny RAR5 entry therefore cannot rent
a 256 MiB or 1 GiB window under the default settings.
These settings are **process-wide**, not per-reader options. Configure them once
at startup if trusted input requires larger dictionaries, for example
`DictionarySizeLimits.Rar5 = 256L << 20;`. Do not temporarily raise them around
untrusted requests. Bound extraction output and concurrency separately; dictionary
limits are not a total-memory budget. See the [API reference](API.md#decompression-dictionary-limits).

View File

@@ -1,5 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Readers;
@@ -47,13 +49,17 @@ public abstract partial class AbstractArchive<TEntry, TVolume>
private async ValueTask EnsureEntriesLoadedAsync()
{
await _lazyEntriesAsync.EnsureFullyLoaded().ConfigureAwait(false);
await foreach (var entry in EntriesAsync.ConfigureAwait(false)) { }
await _lazyVolumesAsync.EnsureFullyLoaded().ConfigureAwait(false);
}
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast()
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast(
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
await foreach (var entry in EntriesAsync.ConfigureAwait(false))
await foreach (
var entry in EntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
)
{
yield return entry;
}

View File

@@ -99,7 +99,8 @@ public abstract partial class AbstractArchive<TEntry, TVolume> : IArchive, IAsyn
private void EnsureEntriesLoaded()
{
_lazyEntries.EnsureFullyLoaded();
// Writable archives expose a shared view rather than this base cache.
Entries.ForEach(_ => { });
_lazyVolumes.EnsureFullyLoaded();
}

View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
@@ -14,17 +15,86 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
where TVolume : IVolume
where TOptions : IWriterOptions
{
// Async property moved from main file
public override IAsyncEnumerable<TEntry> EntriesAsync => GetCurrentEntriesAsync();
private async IAsyncEnumerable<TEntry> GetCurrentEntriesAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
await foreach (
var entry in OldEntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
)
{
cancellationToken.ThrowIfCancellationRequested();
yield return entry;
}
cancellationToken.ThrowIfCancellationRequested();
foreach (var entry in newEntries)
{
cancellationToken.ThrowIfCancellationRequested();
if (!removedEntries.Contains(entry))
{
yield return entry;
}
}
}
private IAsyncEnumerable<TEntry> OldEntriesAsync =>
base.EntriesAsync.Where(x => !removedEntries.Contains(x));
LoadOriginalEntriesAsync().Where(x => !removedEntries.Contains(x));
private async IAsyncEnumerable<TEntry> LoadOriginalEntriesAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
var index = 0;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
while (index < originalEntries.Count)
{
cancellationToken.ThrowIfCancellationRequested();
yield return originalEntries[index++];
}
if (originalEntriesLoaded)
{
yield break;
}
// The cached loader outlives individual enumerations and must not
// retain a caller's cancellation token after that enumeration ends.
originalEntriesAsyncSource ??= base.EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
while (await originalEntriesAsyncSource.MoveNextAsync().ConfigureAwait(false))
{
if (originalEntriesAsyncRead++ < originalEntries.Count)
{
cancellationToken.ThrowIfCancellationRequested();
continue;
}
originalEntries.Add(originalEntriesAsyncSource.Current);
break;
}
if (index == originalEntries.Count)
{
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
originalEntriesAsyncSource = null;
originalEntriesLoaded = true;
}
}
}
private async ValueTask RebuildModifiedCollectionAsync()
{
if (pauseRebuilding)
{
// Disposing the synchronous pause must not perform synchronous I/O for async callers.
asyncRebuildPending = true;
return;
}
hasModifications = true;
asyncRebuildPending = false;
newEntries.RemoveAll(v => removedEntries.Contains(v));
modifiedEntries.Clear();
await foreach (var entry in OldEntriesAsync.ConfigureAwait(false))
@@ -49,20 +119,21 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
)
{
await foreach (
var entry in EntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
var entry in OldEntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
)
{
var path = entry.Key;
if (path is null)
if (entry.Key is { } path && KeysMatch(path, key))
{
continue;
return true;
}
var p = path.Replace('/', '\\');
if (p.Length > 0 && p[0] == '\\')
}
foreach (var entry in newEntries)
{
cancellationToken.ThrowIfCancellationRequested();
if (!removedEntries.Contains(entry) && entry.Key is { } path && KeysMatch(path, key))
{
p = p.Substring(1);
return true;
}
return string.Equals(p, key, StringComparison.OrdinalIgnoreCase);
}
return false;
}
@@ -116,15 +187,24 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
CancellationToken cancellationToken = default
)
{
var activeNewEntries = newEntries.Where(x => !removedEntries.Contains(x)).ToList();
//reset streams of new entries
newEntries.Cast<IWritableArchiveEntry>().ForEach(x => x.Stream.Seek(0, SeekOrigin.Begin));
await SaveToAsync(stream, options, OldEntriesAsync, newEntries, cancellationToken)
activeNewEntries
.Cast<IWritableArchiveEntry>()
.ForEach(x => x.Stream.Seek(0, SeekOrigin.Begin));
await SaveToAsync(stream, options, OldEntriesAsync, activeNewEntries, cancellationToken)
.ConfigureAwait(false);
}
public override async ValueTask DisposeAsync()
{
await base.DisposeAsync().ConfigureAwait(false);
originalEntriesSource?.Dispose();
if (originalEntriesAsyncSource is not null)
{
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
}
originalEntries.Cast<Entry>().ForEach(x => x.Close());
newEntries.Cast<Entry>().ForEach(x => x.Close());
removedEntries.Cast<Entry>().ForEach(x => x.Close());
modifiedEntries.Cast<Entry>().ForEach(x => x.Close());

View File

@@ -32,32 +32,54 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
public void Dispose()
{
archive.pauseRebuilding = false;
archive.RebuildModifiedCollection();
if (!archive.asyncRebuildPending)
{
archive.RebuildModifiedCollection();
}
}
}
private readonly List<TEntry> newEntries = new();
private readonly List<TEntry> removedEntries = new();
private readonly List<TEntry> originalEntries = new();
private readonly LazyReadOnlyCollection<TEntry> originalEntriesView;
private bool originalEntriesLoaded;
private IEnumerator<TEntry>? originalEntriesSource;
private IAsyncEnumerator<TEntry>? originalEntriesAsyncSource;
private int originalEntriesRead;
private int originalEntriesAsyncRead;
private readonly List<TEntry> modifiedEntries = new();
private bool hasModifications;
private bool pauseRebuilding;
private bool asyncRebuildPending;
internal AbstractWritableArchive(ArchiveType type)
: base(type) { }
: base(type)
{
originalEntriesLoaded = true;
originalEntriesView = new LazyReadOnlyCollection<TEntry>(LoadOriginalEntries());
}
internal AbstractWritableArchive(ArchiveType type, SourceStream sourceStream)
: base(type, sourceStream) { }
: base(type, sourceStream)
{
originalEntriesView = new LazyReadOnlyCollection<TEntry>(LoadOriginalEntries());
}
public override ICollection<TEntry> Entries
{
get
{
if (asyncRebuildPending && !pauseRebuilding)
{
RebuildModifiedCollection();
}
if (hasModifications)
{
return modifiedEntries;
}
return base.Entries;
return originalEntriesView;
}
}
@@ -70,12 +92,50 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
return;
}
hasModifications = true;
asyncRebuildPending = false;
newEntries.RemoveAll(v => removedEntries.Contains(v));
modifiedEntries.Clear();
modifiedEntries.AddRange(OldEntries.Concat(newEntries));
}
private IEnumerable<TEntry> OldEntries => base.Entries.Where(x => !removedEntries.Contains(x));
private IEnumerable<TEntry> LoadOriginalEntries()
{
var index = 0;
while (true)
{
while (index < originalEntries.Count)
{
yield return originalEntries[index++];
}
if (originalEntriesLoaded)
{
yield break;
}
// Resume the base cache instead of reparsing its prefix on every API
// switch. Each loader scans at most once; only the first entry object
// seen at each index is exposed to callers and removal checks.
originalEntriesSource ??= base.Entries.GetEnumerator();
while (originalEntriesSource.MoveNext())
{
if (originalEntriesRead++ < originalEntries.Count)
{
continue;
}
originalEntries.Add(originalEntriesSource.Current);
break;
}
if (index == originalEntries.Count)
{
originalEntriesSource.Dispose();
originalEntriesSource = null;
originalEntriesLoaded = true;
}
}
}
private IEnumerable<TEntry> OldEntries =>
originalEntriesView.Where(x => !removedEntries.Contains(x));
public void RemoveEntry(TEntry entry)
{
@@ -126,22 +186,44 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
private bool DoesKeyMatchExisting(string key)
{
foreach (var path in Entries.Select(x => x.Key))
foreach (
var path in OldEntries
.Concat(newEntries)
.Where(x => !removedEntries.Contains(x))
.Select(x => x.Key)
)
{
if (path is null)
if (path is not null && KeysMatch(path, key))
{
continue;
return true;
}
var p = path.Replace('/', '\\');
if (p.Length > 0 && p[0] == '\\')
{
p = p.Substring(1);
}
return string.Equals(p, key, StringComparison.OrdinalIgnoreCase);
}
return false;
}
private bool KeysMatch(string left, string right) =>
string.Equals(NormalizeKey(left), NormalizeKey(right), StringComparison.OrdinalIgnoreCase);
private string NormalizeKey(string key)
{
key = key.Replace('\\', '/');
if (Type is ArchiveType.Zip or ArchiveType.Tar)
{
// ZIP and TAR writers remove drive prefixes and surrounding slashes from names.
#if LEGACY_DOTNET
var colon = key.IndexOf(':');
#else
var colon = key.IndexOf(':', StringComparison.Ordinal);
#endif
if (colon >= 0)
{
key = key.Substring(colon + 1);
}
return key.Trim('/');
}
return key.Length > 0 && key[0] == '/' ? key.Substring(1) : key;
}
ValueTask IWritableAsyncArchive.RemoveEntryAsync(IArchiveEntry entry) =>
RemoveEntryAsync((TEntry)entry);
@@ -180,9 +262,12 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
public void SaveTo(Stream stream, TOptions options)
{
var activeNewEntries = newEntries.Where(x => !removedEntries.Contains(x)).ToList();
//reset streams of new entries
newEntries.Cast<IWritableArchiveEntry>().ForEach(x => x.Stream.Seek(0, SeekOrigin.Begin));
SaveTo(stream, options, OldEntries, newEntries);
activeNewEntries
.Cast<IWritableArchiveEntry>()
.ForEach(x => x.Stream.Seek(0, SeekOrigin.Begin));
SaveTo(stream, options, OldEntries, activeNewEntries);
}
protected TEntry CreateEntry(
@@ -222,6 +307,8 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
public override void Dispose()
{
base.Dispose();
originalEntriesSource?.Dispose();
originalEntries.Cast<Entry>().ForEach(x => x.Close());
newEntries.Cast<Entry>().ForEach(x => x.Close());
removedEntries.Cast<Entry>().ForEach(x => x.Close());
modifiedEntries.Cast<Entry>().ForEach(x => x.Close());

View File

@@ -110,6 +110,7 @@ public partial class GZipArchive
)
{
var stream = (await volumes.SingleAsync().ConfigureAwait(false)).Stream;
stream.Position = headerPosition;
yield return new GZipArchiveEntry(
this,
await GZipFilePart

View File

@@ -15,8 +15,10 @@ namespace SharpCompress.Archives.GZip;
public partial class GZipArchive
: AbstractWritableArchive<GZipArchiveEntry, GZipVolume, GZipWriterOptions>
{
private readonly long headerPosition;
private GZipArchive(SourceStream sourceStream)
: base(ArchiveType.GZip, sourceStream) { }
: base(ArchiveType.GZip, sourceStream) => headerPosition = sourceStream.Position;
internal GZipArchive()
: base(ArchiveType.GZip) { }
@@ -49,6 +51,8 @@ public partial class GZipArchive
{
// Kept hand-written: the generator cannot map IAsyncEnumerable.SingleAsync to LINQ Single.
var stream = volumes.Single().Stream;
// The other API may already have consumed the header or extracted the entry.
stream.Position = headerPosition;
yield return new GZipArchiveEntry(
this,
GZipFilePart.Create(stream, ReaderOptions.ArchiveEncoding, ReaderOptions.Providers),

View File

@@ -196,8 +196,12 @@ public partial class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, Sev
);
}
return CreateEntryStream(
new ReadOnlySubStream(_currentFolderStream, entry.Size, leaveOpen: true)
// Entry boundaries do not end a solid folder's decoder. Its buffered input must
// remain intact until the reader advances to another folder or is disposed.
return new EntryStream(
this,
new ReadOnlySubStream(_currentFolderStream, entry.Size, leaveOpen: true),
rewindDecoderOnDispose: false
);
}

View File

@@ -0,0 +1,95 @@
using System;
using System.Threading;
namespace SharpCompress.Common;
/// <summary>
/// Process-wide dictionary/window and PPMd model size limits, in bytes.
/// Configure once at application startup, not for individual extraction requests.
/// These limits do not bound total memory, allocator overhead, or decompressed output.
/// </summary>
public static class DictionarySizeLimits
{
private static long rar4 = 4L << 20;
private static long rar5 = 64L << 20;
private static long lzma = 64L << 20;
private static long lzma2 = 64L << 20;
private static long ppmd = 64L << 20;
private static long zStandard = 128L << 20;
/// <summary>RAR4 window limit. Defaults to the format maximum of 4 MiB.</summary>
public static long Rar4
{
get => Interlocked.Read(ref rar4);
set => SetMaximum(ref rar4, value, nameof(Rar4));
}
/// <summary>RAR5 window limit. Defaults to 64 MiB.</summary>
public static long Rar5
{
get => Interlocked.Read(ref rar5);
set => SetMaximum(ref rar5, value, nameof(Rar5));
}
/// <summary>LZMA dictionary limit, including LZip, ZIP, and 7z. Defaults to 64 MiB.</summary>
public static long Lzma
{
get => Interlocked.Read(ref lzma);
set => SetMaximum(ref lzma, value, nameof(Lzma));
}
/// <summary>LZMA2 dictionary limit, including XZ and 7z. Defaults to 64 MiB.</summary>
public static long Lzma2
{
get => Interlocked.Read(ref lzma2);
set => SetMaximum(ref lzma2, value, nameof(Lzma2));
}
/// <summary>PPMd model limit, including ZIP, 7z, and RAR models. Defaults to 64 MiB.</summary>
public static long Ppmd
{
get => Interlocked.Read(ref ppmd);
set => SetMaximum(ref ppmd, value, nameof(Ppmd));
}
/// <summary>
/// Zstandard streaming window limit. Defaults to 128 MiB.
/// Must be between 1 KiB and 1 GiB on 32-bit processes or 2 GiB on 64-bit processes.
/// </summary>
public static long ZStandard
{
get => Interlocked.Read(ref zStandard);
set
{
// Match the engine bounds before replacing a working process-wide configuration.
var maximum = 1L << (IntPtr.Size == 4 ? 30 : 31);
if (value < 1024 || value > maximum)
{
throw new ArgumentOutOfRangeException(
nameof(ZStandard),
$"Zstandard window limits must be between 1024 and {maximum} bytes."
);
}
SetMaximum(ref zStandard, value, nameof(ZStandard));
}
}
private static void SetMaximum(ref long field, long value, string name)
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException(name, "Dictionary size limits must be positive.");
}
Interlocked.Exchange(ref field, value);
}
internal static void Validate(long size, long maximum, string format)
{
if (size <= 0 || size > maximum)
{
throw new InvalidFormatException(
$"{format} dictionary size {size} is invalid or exceeds the maximum of {maximum} bytes."
);
}
}
}

View File

@@ -31,7 +31,7 @@ public partial class EntryStream
}
//Need a safe standard approach to this - it's okay for compression to overreads. Handling needs to be standardised
if (_stream is IStreamStack ss)
if (_rewindDecoderOnDispose && _stream is IStreamStack ss)
{
if (ss.BaseStream() is SharpCompress.Compressors.Deflate.DeflateStream deflateStream)
{

View File

@@ -12,13 +12,15 @@ public partial class EntryStream : AsyncDisposableStream
{
private readonly IReader _reader;
private readonly Stream _stream;
private readonly bool _rewindDecoderOnDispose;
private bool _completed;
private bool _isDisposed;
internal EntryStream(IReader reader, Stream stream)
internal EntryStream(IReader reader, Stream stream, bool rewindDecoderOnDispose = true)
{
_reader = reader;
_stream = stream;
_rewindDecoderOnDispose = rewindDecoderOnDispose;
}
protected override void Dispose(bool disposing)
@@ -34,7 +36,7 @@ public partial class EntryStream : AsyncDisposableStream
}
//Need a safe standard approach to this - it's okay for compression to overreads. Handling needs to be standardised
if (_stream is IStreamStack ss)
if (_rewindDecoderOnDispose && _stream is IStreamStack ss)
{
if (
ss.GetStream<SharpCompress.Compressors.Deflate.DeflateStream>()

View File

@@ -128,20 +128,22 @@ internal sealed partial class ArchiveReader
streamSwitch.Set(this, header);
var type = ReadId();
if (type != BlockType.Header)
if (type is not BlockType.Header and not BlockType.EncodedHeader)
{
if (type != BlockType.EncodedHeader)
{
throw new ArchiveOperationException();
}
var dataVector = await ReadAndDecodePackedStreamsAsync(
db._startPositionAfterHeader,
db.PasswordProvider,
cancellationToken
)
.ConfigureAwait(false);
throw new ArchiveOperationException();
}
using var dataVector =
type == BlockType.EncodedHeader
? await ReadAndDecodePackedStreamsAsync(
db._startPositionAfterHeader,
db.PasswordProvider,
cancellationToken
)
.ConfigureAwait(false)
: null;
if (dataVector is not null)
{
// compressed header without content is odd but ok
if (dataVector.Count == 0)
{
@@ -168,12 +170,14 @@ internal sealed partial class ArchiveReader
return db;
}
private async ValueTask<List<byte[]>> ReadAndDecodePackedStreamsAsync(
private async ValueTask<DecodedMetadataStreams> ReadAndDecodePackedStreamsAsync(
long baseOffset,
IPasswordProvider pass,
CancellationToken cancellationToken
)
{
DecodedMetadataStreams? dataVector = null;
var buffer = decoderBufferPool.Rent(MetadataReadBufferSize);
try
{
ReadStreamsInfo(
@@ -189,10 +193,11 @@ internal sealed partial class ArchiveReader
dataStartPos += baseOffset;
var dataVector = new List<byte[]>(folders.Count);
dataVector = new DecodedMetadataStreams(folders.Count);
var packIndex = 0;
foreach (var folder in folders)
{
var unpackSize = GetMetadataSize(folder);
var oldDataStartPos = dataStartPos;
var myPackSizes = new long[folder._packStreams.Count];
for (var i = 0; i < myPackSizes.Length; i++)
@@ -202,7 +207,7 @@ internal sealed partial class ArchiveReader
dataStartPos += packSize;
}
var outStream = await DecoderStreamHelper
using var outStream = await DecoderStreamHelper
.CreateDecoderStreamAsync(
_stream,
oldDataStartPos,
@@ -217,27 +222,36 @@ internal sealed partial class ArchiveReader
// header fails; its source stream remains borrowed.
await using var decoderScope = outStream.DisposeAsyncScope().ConfigureAwait(false);
var unpackSize = checked((int)folder.GetUnpackSize());
var data = new byte[unpackSize];
await outStream
.ReadExactAsync(data, 0, data.Length, cancellationToken)
.ConfigureAwait(false);
if (
// Grow only as the decoder actually produces metadata, never
// from the untrusted unpack-size declaration alone.
var decoded = new PooledMemoryStream(0, Constants.BufferSize, decoderBufferPool);
dataVector.Add(decoded);
var crc = Crc.INIT_CRC;
for (var remaining = unpackSize; remaining > 0; )
{
var count = Math.Min(MetadataReadBufferSize, remaining);
await outStream
.ReadAsync(new byte[1], 0, 1, cancellationToken)
.ConfigureAwait(false) > 0
.ReadExactAsync(buffer, 0, count, cancellationToken)
.ConfigureAwait(false);
await decoded
.WriteAsync(buffer, 0, count, cancellationToken)
.ConfigureAwait(false);
if (folder.UnpackCrcDefined)
{
crc = Crc.Update(crc, buffer, 0, count);
}
remaining -= count;
}
if (
await outStream.ReadAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false)
!= 0
)
{
throw new InvalidFormatException("Decoded stream is longer than expected.");
}
dataVector.Add(data);
if (folder.UnpackCrcDefined)
{
if (
Crc.Finish(Crc.Update(Crc.INIT_CRC, data, 0, unpackSize))
!= folder._unpackCrc
)
if (Crc.Finish(crc) != folder._unpackCrc)
{
var exception = new InvalidFormatException(
"Decoded stream does not match expected CRC."
@@ -250,7 +264,15 @@ internal sealed partial class ArchiveReader
}
return dataVector;
}
finally { }
catch
{
dataVector?.Dispose();
throw;
}
finally
{
decoderBufferPool.Return(buffer);
}
}
private async ValueTask ReadHeaderAsync(
@@ -269,15 +291,17 @@ internal sealed partial class ArchiveReader
type = ReadId();
}
List<byte[]>? dataVector = null;
if (type == BlockType.AdditionalStreamsInfo)
using var dataVector =
type == BlockType.AdditionalStreamsInfo
? await ReadAndDecodePackedStreamsAsync(
db._startPositionAfterHeader,
getTextPassword,
cancellationToken
)
.ConfigureAwait(false)
: null;
if (dataVector is not null)
{
dataVector = await ReadAndDecodePackedStreamsAsync(
db._startPositionAfterHeader,
getTextPassword,
cancellationToken
)
.ConfigureAwait(false);
type = ReadId();
}
@@ -327,6 +351,11 @@ internal sealed partial class ArchiveReader
}
var numFiles = ReadNum();
ValidateFileCount(
numFiles,
unpackSizes.Count,
db._majorVersion > 0 || db._minorVersion > 2
);
db._files = new List<CFileItem>(numFiles);
for (var i = 0; i < numFiles; i++)
{
@@ -353,7 +382,7 @@ internal sealed partial class ArchiveReader
case BlockType.Name:
using (var streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector ?? []);
streamSwitch.Set(this, dataVector);
for (var i = 0; i < db._files.Count; i++)
{
db._files[i].Name = _currentReader.ReadString();

View File

@@ -16,6 +16,7 @@ namespace SharpCompress.Common.SevenZip;
internal partial class ArchiveReader
{
private const int MetadataReadBufferSize = 8192;
private readonly ArrayPool<byte> decoderBufferPool;
internal ArchiveReader(ArrayPool<byte>? decoderBufferPool = null) =>
@@ -36,6 +37,12 @@ internal partial class ArchiveReader
_currentReader = new DataReader(buffer, offset, length);
}
internal void AddByteStream(PooledMemoryStream buffer)
{
_readerStack.Push(_currentReader);
_currentReader = new DataReader(buffer);
}
internal void DeleteByteStream() => _currentReader = _readerStack.Pop();
#region Private Methods - Data Reader
@@ -98,6 +105,7 @@ internal partial class ArchiveReader
private BitVector ReadBitVector(int length)
{
_currentReader.RequireBytes(((long)length + 7) / 8);
var bits = new BitVector(length);
byte data = 0;
@@ -133,12 +141,16 @@ internal partial class ArchiveReader
return ReadBitVector(length);
}
private void ReadNumberVector(List<byte[]>? dataVector, int numFiles, Action<int, long?> action)
private void ReadNumberVector(
List<PooledMemoryStream>? dataVector,
int numFiles,
Action<int, long?> action
)
{
var defined = ReadOptionalBitVector(numFiles);
using var streamSwitch = new CStreamSwitch();
streamSwitch.Set(this, dataVector ?? []);
streamSwitch.Set(this, dataVector);
for (var i = 0; i < numFiles; i++)
{
@@ -167,7 +179,7 @@ internal partial class ArchiveReader
}
private void ReadDateTimeVector(
List<byte[]>? dataVector,
List<PooledMemoryStream>? dataVector,
int numFiles,
Action<int, DateTime?> action
) =>
@@ -178,14 +190,14 @@ internal partial class ArchiveReader
);
private void ReadAttributeVector(
List<byte[]>? dataVector,
List<PooledMemoryStream>? dataVector,
int numFiles,
Action<int, uint?> action
)
{
var boolVector = ReadOptionalBitVector(numFiles);
using var streamSwitch = new CStreamSwitch();
streamSwitch.Set(this, dataVector ?? []);
streamSwitch.Set(this, dataVector);
for (var i = 0; i < numFiles; i++)
{
if (boolVector[i])
@@ -208,9 +220,16 @@ internal partial class ArchiveReader
try
{
var numCoders = ReadNum();
// CFolder.CheckStructure supports at most 32 coders/bind pairs, but runs
// after parsing. Enforce those limits before allocating their lists.
if (numCoders == 0 || numCoders > CFolder.MaxCoders)
{
throw new InvalidFormatException("Invalid number of 7z coders.");
}
_currentReader.RequireBytes(numCoders);
folder._coders = new List<CCoderInfo>(numCoders);
var numInStreams = 0;
var numOutStreams = 0;
long numInStreamsTotal = 0;
long numOutStreamsTotal = 0;
for (var i = 0; i < numCoders; i++)
{
try
@@ -247,6 +266,7 @@ internal partial class ArchiveReader
if ((mainByte & 0x20) != 0)
{
var propsSize = ReadNum();
_currentReader.RequireBytes(propsSize);
coder._props = new byte[propsSize];
ReadBytes(coder._props, 0, propsSize);
}
@@ -256,30 +276,48 @@ internal partial class ArchiveReader
throw new NotSupportedException();
}
numInStreams += coder._numInStreams;
numOutStreams += coder._numOutStreams;
if (coder._numInStreams == 0 || coder._numOutStreams == 0)
{
throw new InvalidFormatException("Invalid 7z coder stream count.");
}
numInStreamsTotal += coder._numInStreams;
numOutStreamsTotal += coder._numOutStreams;
}
finally { }
}
if (
numOutStreamsTotal - 1 > CFolder.MaxBindPairs
|| numInStreamsTotal > int.MaxValue
|| numInStreamsTotal < numOutStreamsTotal
)
{
throw new InvalidFormatException("Invalid 7z folder stream counts.");
}
var numInStreams = (int)numInStreamsTotal;
var numOutStreams = (int)numOutStreamsTotal;
var numBindPairs = numOutStreams - 1;
_currentReader.RequireBytes(2L * numBindPairs);
folder._bindPairs = new List<CBindPair>(numBindPairs);
for (var i = 0; i < numBindPairs; i++)
{
var bp = new CBindPair();
bp._inIndex = ReadNum();
bp._outIndex = ReadNum();
if (
bp._inIndex >= numInStreams
|| bp._outIndex >= numOutStreams
|| folder.FindBindPairForInStream(bp._inIndex) >= 0
|| folder.FindBindPairForOutStream(bp._outIndex) >= 0
)
{
throw new InvalidFormatException("Invalid 7z bind pair.");
}
folder._bindPairs.Add(bp);
}
if (numInStreams < numBindPairs)
{
throw new NotSupportedException();
}
var numPackStreams = numInStreams - numBindPairs;
//folder.PackStreams.Reserve(numPackStreams);
if (numPackStreams == 1)
{
for (var i = 0; i < numInStreams; i++)
@@ -298,9 +336,19 @@ internal partial class ArchiveReader
}
else
{
_currentReader.RequireBytes(numPackStreams);
var packStreams = new HashSet<int>();
for (var i = 0; i < numPackStreams; i++)
{
var num = ReadNum();
if (
num >= numInStreams
|| folder.FindBindPairForInStream(num) >= 0
|| !packStreams.Add(num)
)
{
throw new InvalidFormatException("Invalid 7z packed stream index.");
}
folder._packStreams.Add(num);
}
}
@@ -310,7 +358,23 @@ internal partial class ArchiveReader
private List<uint?> ReadHashDigests(int count)
{
// An all-defined vector has no bitmap, but must have a CRC per item.
// Check it before ReadOptionalBitVector allocates the implicit bitmap.
var reader = _currentReader.Clone();
if (reader.ReadByte() != 0)
{
reader.RequireBytes(4L * count);
}
var defined = ReadOptionalBitVector(count);
long numDefined = 0;
for (var i = 0; i < count; i++)
{
if (defined[i])
{
numDefined++;
}
}
_currentReader.RequireBytes(4 * numDefined);
var digests = new List<uint?>(count);
for (var i = 0; i < count; i++)
{
@@ -342,6 +406,7 @@ internal partial class ArchiveReader
var numPackStreams = ReadNum();
WaitAttribute(BlockType.Size);
_currentReader.RequireBytes(numPackStreams);
packSizes = new List<long>(numPackStreams);
for (var i = 0; i < numPackStreams; i++)
{
@@ -377,7 +442,7 @@ internal partial class ArchiveReader
finally { }
}
private void ReadUnpackInfo(List<byte[]>? dataVector, out List<CFolder> folders)
private void ReadUnpackInfo(List<PooledMemoryStream>? dataVector, out List<CFolder> folders)
{
try
{
@@ -386,10 +451,11 @@ internal partial class ArchiveReader
using (var streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector ?? []);
streamSwitch.Set(this, dataVector);
//folders.Clear();
//folders.Reserve(numFolders);
// Folder definitions may live in an external metadata stream.
// Validate against the selected stream, not the outer header.
_currentReader.RequireBytes(numFolders);
folders = new List<CFolder>(numFolders);
var index = 0;
for (var i = 0; i < numFolders; i++)
@@ -411,6 +477,10 @@ internal partial class ArchiveReader
var size = checked((long)ReadNumber());
folder._unpackSizes.Add(size);
}
if (!folder.CheckStructure())
{
throw new InvalidFormatException("Invalid 7z folder structure.");
}
}
for (; ; )
@@ -482,6 +552,21 @@ internal partial class ArchiveReader
}
}
// Every substream except the last in each folder needs an explicit
// size. Reject impossible counts before loops or digest allocations.
long numExplicitSizes = 0;
long numStreamsTotal = 0;
foreach (var count in numUnpackStreamsInFolders)
{
numExplicitSizes += Math.Max(0, count - 1);
numStreamsTotal += count;
}
if (numStreamsTotal > int.MaxValue || (numExplicitSizes != 0 && type != BlockType.Size))
{
throw new InvalidFormatException("Invalid 7z substream count.");
}
_currentReader.RequireBytes(numExplicitSizes);
unpackSizes = new List<long>(folders.Count);
for (var i = 0; i < numUnpackStreamsInFolders.Count; i++)
{
@@ -498,6 +583,10 @@ internal partial class ArchiveReader
if (type == BlockType.Size)
{
var size = checked((long)ReadNumber());
if (size > folders[i].GetUnpackSize() - sum)
{
throw new InvalidFormatException("Invalid 7z substream size.");
}
unpackSizes.Add(size);
sum += size;
}
@@ -578,7 +667,7 @@ internal partial class ArchiveReader
}
private void ReadStreamsInfo(
List<byte[]>? dataVector,
List<PooledMemoryStream>? dataVector,
out long dataOffset,
out List<long> packSizes,
out List<uint?> packCrCs,
@@ -626,8 +715,98 @@ internal partial class ArchiveReader
finally { }
}
private List<byte[]> ReadAndDecodePackedStreams(long baseOffset, IPasswordProvider pass)
private static int GetMetadataSize(CFolder folder)
{
var size = folder.GetUnpackSize();
if (size < 0 || size > int.MaxValue)
{
throw new InvalidFormatException("7z metadata is too large.");
}
return (int)size;
}
private void ValidateFileCount(int numFiles, int numStreams, bool checkRecordsSize)
{
if (!checkRecordsSize)
{
// Before 0.3, known properties may have inaccurate sizes, so walking
// them by their declared lengths would disagree with the real parser.
// Bound the allocation by the minimum possible empty-stream bitmap;
// ReadBitVector separately bounds the actual bitmap at its read position.
var minimumEmptyStreams = Math.Max(0L, (long)numFiles - numStreams);
_currentReader.RequireBytes((minimumEmptyStreams + 7) / 8);
return;
}
// Nonempty files correspond to already parsed substreams. Empty files
// require one bit each in kEmptyStream, not one byte per file. Preflight
// that property before allocating CFileItem objects or implicit vectors.
var reader = _currentReader.Clone();
var numEmptyStreams = 0;
var hasEmptyStreams = false;
for (; ; )
{
var type = reader.ReadNumber();
if (type == (ulong)BlockType.End)
{
break;
}
var size = reader.ReadNumber();
if (size > (ulong)reader.Remaining)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
}
var start = reader.Offset;
if (type == (ulong)BlockType.EmptyStream)
{
var bitmapSize = ((long)numFiles + 7) / 8;
if (hasEmptyStreams || size < (ulong)bitmapSize)
{
throw new InvalidFormatException("Invalid 7z empty stream bitmap.");
}
hasEmptyStreams = true;
byte bits = 0;
for (var i = 0; i < numFiles; i++)
{
if ((i & 7) == 0)
{
bits = reader.ReadByte();
}
if ((bits & 0x80) != 0)
{
numEmptyStreams++;
}
bits <<= 1;
}
}
reader.SkipData((long)size - (reader.Offset - start));
}
if ((long)numStreams + numEmptyStreams != numFiles)
{
throw new InvalidFormatException("7z file count does not match the stream count.");
}
}
private sealed class DecodedMetadataStreams(int capacity)
: List<PooledMemoryStream>(capacity),
IDisposable
{
public void Dispose()
{
foreach (var stream in this)
{
stream.Dispose();
}
Clear();
}
}
private DecodedMetadataStreams ReadAndDecodePackedStreams(
long baseOffset,
IPasswordProvider pass
)
{
DecodedMetadataStreams? dataVector = null;
var buffer = decoderBufferPool.Rent(MetadataReadBufferSize);
try
{
ReadStreamsInfo(
@@ -643,10 +822,11 @@ internal partial class ArchiveReader
dataStartPos += baseOffset;
var dataVector = new List<byte[]>(folders.Count);
dataVector = new DecodedMetadataStreams(folders.Count);
var packIndex = 0;
foreach (var folder in folders)
{
var unpackSize = GetMetadataSize(folder);
var oldDataStartPos = dataStartPos;
var myPackSizes = new long[folder._packStreams.Count];
for (var i = 0; i < myPackSizes.Length; i++)
@@ -667,21 +847,29 @@ internal partial class ArchiveReader
decoderBufferPool
);
var unpackSize = checked((int)folder.GetUnpackSize());
var data = new byte[unpackSize];
outStream.ReadExact(data, 0, data.Length);
// Do not reserve the declared unpack size: malformed compressed
// metadata can advertise GiB of output without producing a byte.
var decoded = new PooledMemoryStream(0, Constants.BufferSize, decoderBufferPool);
dataVector.Add(decoded);
var crc = Crc.INIT_CRC;
for (var remaining = unpackSize; remaining > 0; )
{
var count = Math.Min(MetadataReadBufferSize, remaining);
outStream.ReadExact(buffer, 0, count);
decoded.Write(buffer, 0, count);
if (folder.UnpackCrcDefined)
{
crc = Crc.Update(crc, buffer, 0, count);
}
remaining -= count;
}
if (outStream.ReadByte() >= 0)
{
throw new InvalidFormatException("Decoded stream is longer than expected.");
}
dataVector.Add(data);
if (folder.UnpackCrcDefined)
{
if (
Crc.Finish(Crc.Update(Crc.INIT_CRC, data, 0, unpackSize))
!= folder._unpackCrc
)
if (Crc.Finish(crc) != folder._unpackCrc)
{
var exception = new InvalidFormatException(
"Decoded stream does not match expected CRC."
@@ -694,7 +882,15 @@ internal partial class ArchiveReader
}
return dataVector;
}
finally { }
catch
{
dataVector?.Dispose();
throw;
}
finally
{
decoderBufferPool.Return(buffer);
}
}
private void ReadHeader(ArchiveDatabase db, IPasswordProvider getTextPassword)
@@ -709,13 +905,12 @@ internal partial class ArchiveReader
type = ReadId();
}
List<byte[]>? dataVector = null;
if (type == BlockType.AdditionalStreamsInfo)
using var dataVector =
type == BlockType.AdditionalStreamsInfo
? ReadAndDecodePackedStreams(db._startPositionAfterHeader, getTextPassword)
: null;
if (dataVector is not null)
{
dataVector = ReadAndDecodePackedStreams(
db._startPositionAfterHeader,
getTextPassword
);
type = ReadId();
}
@@ -765,6 +960,11 @@ internal partial class ArchiveReader
}
var numFiles = ReadNum();
ValidateFileCount(
numFiles,
unpackSizes.Count,
db._majorVersion > 0 || db._minorVersion > 2
);
db._files = new List<CFileItem>(numFiles);
for (var i = 0; i < numFiles; i++)
{
@@ -791,7 +991,7 @@ internal partial class ArchiveReader
case BlockType.Name:
using (var streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector ?? []);
streamSwitch.Set(this, dataVector);
for (var i = 0; i < db._files.Count; i++)
{
db._files[i].Name = _currentReader.ReadString();
@@ -1077,18 +1277,18 @@ internal partial class ArchiveReader
streamSwitch.Set(this, header);
var type = ReadId();
if (type != BlockType.Header)
if (type is not BlockType.Header and not BlockType.EncodedHeader)
{
if (type != BlockType.EncodedHeader)
{
throw new ArchiveOperationException();
}
var dataVector = ReadAndDecodePackedStreams(
db._startPositionAfterHeader,
db.PasswordProvider
);
throw new ArchiveOperationException();
}
// Keep the pooled metadata alive through parsing, including on errors.
using var dataVector =
type == BlockType.EncodedHeader
? ReadAndDecodePackedStreams(db._startPositionAfterHeader, db.PasswordProvider)
: null;
if (dataVector is not null)
{
// compressed header without content is odd but ok
if (dataVector.Count == 0)
{

View File

@@ -6,6 +6,9 @@ namespace SharpCompress.Common.SevenZip;
internal class CFolder
{
internal const int MaxCoders = 32;
internal const int MaxBindPairs = 32;
internal List<CCoderInfo> _coders = new();
internal List<CBindPair> _bindPairs = new();
internal List<int> _packStreams = new();
@@ -98,11 +101,9 @@ internal class CFolder
public bool CheckStructure()
{
const int kNumCodersMax = 32; // don't change it
const int kMaskSize = 32; // it must be >= kNumCodersMax
const int kNumBindsMax = 32;
const int kMaskSize = 32; // it must be >= MaxCoders
if (_coders.Count > kNumCodersMax || _bindPairs.Count > kNumBindsMax)
if (_coders.Count > MaxCoders || _bindPairs.Count > MaxBindPairs)
{
return false;
}

View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using SharpCompress.Compressors.LZMA;
using SharpCompress.IO;
namespace SharpCompress.Common.SevenZip;
@@ -33,7 +34,16 @@ internal struct CStreamSwitch : IDisposable
_active = true;
}
public void Set(ArchiveReader archive, List<byte[]> dataVector)
public void Set(ArchiveReader archive, PooledMemoryStream data)
{
Dispose();
_archive = archive;
_archive.AddByteStream(data);
_needRemove = true;
_active = true;
}
public void Set(ArchiveReader archive, IReadOnlyList<PooledMemoryStream>? dataVector)
{
Dispose();
_active = true;
@@ -42,13 +52,13 @@ internal struct CStreamSwitch : IDisposable
if (external != 0)
{
var dataIndex = archive.ReadNum();
if (dataIndex < 0 || dataIndex >= dataVector.Count)
if (dataVector is null || dataIndex < 0 || dataIndex >= dataVector.Count)
{
throw new ArchiveOperationException();
}
_archive = archive;
_archive.AddByteStream(dataVector[dataIndex], 0, dataVector[dataIndex].Length);
_archive.AddByteStream(dataVector[dataIndex]);
_needRemove = true;
_active = true;
}

View File

@@ -2,6 +2,7 @@
using System.IO;
using System.Text;
using SharpCompress.Compressors.LZMA;
using SharpCompress.IO;
namespace SharpCompress.Common.SevenZip;
@@ -29,7 +30,8 @@ internal class DataReader
#region Variables
private readonly byte[] _buffer;
private readonly byte[]? _buffer;
private readonly PooledMemoryStream? _pooledBuffer;
private readonly int _ending;
#endregion
@@ -43,8 +45,37 @@ internal class DataReader
_ending = offset + length;
}
public DataReader(PooledMemoryStream buffer)
{
_pooledBuffer = buffer;
_ending = checked((int)buffer.Length);
}
private DataReader(DataReader source)
{
_buffer = source._buffer;
_pooledBuffer = source._pooledBuffer;
_ending = source._ending;
Offset = source.Offset;
}
public int Offset { get; private set; }
public int Remaining => _ending - Offset;
public DataReader Clone() => new(this);
private byte GetByte(int offset) =>
_buffer is not null ? _buffer[offset] : _pooledBuffer!.ReadByteAt(offset);
public void RequireBytes(long count)
{
if (count < 0 || count > Remaining)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
}
}
public byte ReadByte()
{
if (Offset >= _ending)
@@ -52,28 +83,28 @@ internal class DataReader
throw new IncompleteArchiveException("Unexpected end of stream.");
}
return _buffer[Offset++];
return GetByte(Offset++);
}
public void ReadBytes(byte[] buffer, int offset, int length)
{
if (length > _ending - Offset)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
}
RequireBytes(length);
if (_pooledBuffer is not null)
{
_pooledBuffer.ReadAt(Offset, buffer, offset, length);
Offset += length;
return;
}
while (length-- > 0)
{
buffer[offset++] = _buffer[Offset++];
buffer[offset++] = _buffer![Offset++];
}
}
public void SkipData(long size)
{
if (size > _ending - Offset)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
}
RequireBytes(size);
Offset += (int)size;
}
@@ -87,7 +118,7 @@ internal class DataReader
throw new IncompleteArchiveException("Unexpected end of stream.");
}
var firstByte = _buffer[Offset++];
var firstByte = GetByte(Offset++);
byte mask = 0x80;
ulong value = 0;
@@ -105,7 +136,7 @@ internal class DataReader
throw new IncompleteArchiveException("Unexpected end of stream.");
}
value |= (ulong)_buffer[Offset++] << (8 * i);
value |= (ulong)GetByte(Offset++) << (8 * i);
mask >>= 1;
}
@@ -125,26 +156,36 @@ internal class DataReader
public uint ReadUInt32()
{
if (Offset + 4 > _ending)
RequireBytes(4);
if (_buffer is not null)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
var res = Get32(_buffer, Offset);
Offset += 4;
return res;
}
var res = Get32(_buffer, Offset);
Offset += 4;
return res;
uint value = 0;
for (var i = 0; i < 4; i++)
{
value |= (uint)ReadByte() << (8 * i);
}
return value;
}
public ulong ReadUInt64()
{
if (Offset + 8 > _ending)
RequireBytes(8);
if (_buffer is not null)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
var res = Get64(_buffer, Offset);
Offset += 8;
return res;
}
var res = Get64(_buffer, Offset);
Offset += 8;
return res;
ulong value = 0;
for (var i = 0; i < 8; i++)
{
value |= (ulong)ReadByte() << (8 * i);
}
return value;
}
public string ReadString()
@@ -153,12 +194,12 @@ internal class DataReader
for (; ; )
{
if (ending + 2 > _ending)
if (_ending - ending < 2)
{
throw new IncompleteArchiveException("Unexpected end of stream.");
}
if (_buffer[ending] == 0 && _buffer[ending + 1] == 0)
if (GetByte(ending) == 0 && GetByte(ending + 1) == 0)
{
break;
}
@@ -166,10 +207,35 @@ internal class DataReader
ending += 2;
}
var str = Encoding.Unicode.GetString(_buffer, Offset, ending - Offset);
var str = _buffer is not null
? Encoding.Unicode.GetString(_buffer, Offset, ending - Offset)
: ReadPooledString(ending - Offset);
Offset = ending + 2;
return str;
}
private string ReadPooledString(int length)
{
if (length == 0)
{
return string.Empty;
}
// Decode bounded chunks, preserving surrogate pairs across block boundaries,
// rather than copying a potentially header-sized name into a temporary array.
var bytes = new byte[Math.Min(length, 256)];
var chars = new char[Encoding.Unicode.GetMaxCharCount(bytes.Length)];
var decoder = Encoding.Unicode.GetDecoder();
var result = new StringBuilder();
for (var offset = 0; offset < length; )
{
var count = Math.Min(bytes.Length, length - offset);
_pooledBuffer!.ReadAt(Offset + offset, bytes, 0, count);
var charCount = decoder.GetChars(bytes, 0, count, chars, 0, offset + count == length);
result.Append(chars, 0, charCount);
offset += count;
}
return result.ToString();
}
#endregion
}

View File

@@ -37,6 +37,7 @@ internal sealed partial class TarHeader
internal IArchiveEncoding ArchiveEncoding { get; }
internal const int BLOCK_SIZE = 512;
internal const long MAX_ENTRY_SIZE = long.MaxValue - (BLOCK_SIZE - 1);
// Maximum size for long name/link headers to prevent memory exhaustion attacks
// This is generous enough for most real-world scenarios (32KB)
@@ -632,14 +633,34 @@ internal sealed partial class TarHeader
private static EntryType ReadEntryType(byte[] buffer) => (EntryType)buffer[156];
private long ReadSize(byte[] buffer)
private static long ReadSize(byte[] buffer)
{
if ((buffer[124] & 0x80) == 0x80) // if size in binary
var size =
(buffer[124] & 0x80) != 0
? ReadBase256Size(buffer)
: ReadAsciiInt64Base8(buffer, 124, 11);
// Negative sizes can rewind seekable archives; oversized sizes overflow padding.
// Validate here so ordinary entries and metadata use the same sync/async guard.
if (size is < 0 or > MAX_ENTRY_SIZE)
{
return BinaryPrimitives.ReadInt64BigEndian(buffer.AsSpan(0x80));
throw new InvalidFormatException($"Invalid TAR entry size: {size}.");
}
return ReadAsciiInt64Base8(buffer, 124, 11);
return size;
}
private static long ReadBase256Size(byte[] buffer)
{
// Only the last eight bytes fit in a long. Do not silently truncate the
// remaining value bits, including the sign bits of GNU negative sizes.
var leadingBits = (buffer[124] & 0x7F) | buffer[125] | buffer[126] | buffer[127];
if (leadingBits != 0)
{
throw new InvalidFormatException("TAR entry size does not fit in 64 bits.");
}
return BinaryPrimitives.ReadInt64BigEndian(buffer.AsSpan(128, 8));
}
private static byte[] ReadBlock(BinaryReader reader)

View File

@@ -21,25 +21,33 @@ internal static partial class TarHeaderFactory
#endif
var globalPaxMetadata = new TarHeader.PaxMetadata();
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
while (true)
{
TarHeader? header = null;
try
{
if (mode == StreamingMode.Seekable)
{
stream.Position = position;
}
header = new TarHeader(archiveEncoding);
if (!await header.ReadAsync(reader, globalPaxMetadata).ConfigureAwait(false))
{
yield break;
}
// PAX metadata can replace the raw size; validate before either mode uses it.
var paddedSize = PadTo512(header.Size);
switch (mode)
{
case StreamingMode.Seekable:
{
header.DataStartPosition = stream.Position;
//skip to nearest 512
stream.Position += PadTo512(header.Size);
// Reject padded sizes or end offsets that would wrap into a backward seek.
stream.Position = checked(stream.Position + paddedSize);
position = stream.Position;
}
break;
case StreamingMode.Streaming:

View File

@@ -14,11 +14,17 @@ internal static partial class TarHeaderFactory
)
{
var globalPaxMetadata = new TarHeader.PaxMetadata();
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
while (true)
{
TarHeader? header = null;
try
{
if (mode == StreamingMode.Seekable)
{
// Another entry reader or loader may have moved the shared stream.
stream.Position = position;
}
var reader = new BinaryReader(stream, archiveEncoding.Default, leaveOpen: false);
header = new TarHeader(archiveEncoding);
@@ -26,14 +32,17 @@ internal static partial class TarHeaderFactory
{
yield break;
}
// PAX metadata can replace the raw size; validate before either mode uses it.
var paddedSize = PadTo512(header.Size);
switch (mode)
{
case StreamingMode.Seekable:
{
header.DataStartPosition = stream.Position;
//skip to nearest 512
stream.Position += PadTo512(header.Size);
// Reject padded sizes or end offsets that would wrap into a backward seek.
stream.Position = checked(stream.Position + paddedSize);
position = stream.Position;
}
break;
case StreamingMode.Streaming:
@@ -59,11 +68,12 @@ internal static partial class TarHeaderFactory
private static long PadTo512(long size)
{
var zeros = (int)(size % 512);
if (zeros == 0)
// PAX size overrides bypass ReadSize, so validate the effective size too.
if (size is < 0 or > TarHeader.MAX_ENTRY_SIZE)
{
return size;
throw new InvalidFormatException($"Invalid TAR entry size: {size}.");
}
return 512 - zeros + size;
return (size + 511) & ~511L;
}
}

View File

@@ -21,26 +21,24 @@ internal partial class CBZip2InputStream
var retChar = currentChar;
switch (currentState)
{
case START_BLOCK_STATE:
break;
case RAND_PART_A_STATE:
break;
case RAND_PART_B_STATE:
await SetupRandPartBAsync(cancellationToken).ConfigureAwait(false);
break;
case RAND_PART_C_STATE:
await SetupRandPartCAsync(cancellationToken).ConfigureAwait(false);
break;
case NO_RAND_PART_A_STATE:
break;
case NO_RAND_PART_B_STATE:
await SetupNoRandPartBAsync(cancellationToken).ConfigureAwait(false);
break;
case NO_RAND_PART_C_STATE:
await SetupNoRandPartCAsync(cancellationToken).ConfigureAwait(false);
break;
case START_BLOCK_STATE:
case RAND_PART_A_STATE:
case NO_RAND_PART_A_STATE:
default:
break;
// Match ReadByte(): retries after a decoding error must not generate endless output.
throw new InvalidFormatException("BZip2: invalid decoder state");
}
return retChar;
}

View File

@@ -263,26 +263,25 @@ internal partial class CBZip2InputStream : Stream
var retChar = currentChar;
switch (currentState)
{
case START_BLOCK_STATE:
break;
case RAND_PART_A_STATE:
break;
case RAND_PART_B_STATE:
SetupRandPartB();
break;
case RAND_PART_C_STATE:
SetupRandPartC();
break;
case NO_RAND_PART_A_STATE:
break;
case NO_RAND_PART_B_STATE:
SetupNoRandPartB();
break;
case NO_RAND_PART_C_STATE:
SetupNoRandPartC();
break;
case START_BLOCK_STATE:
case RAND_PART_A_STATE:
case NO_RAND_PART_A_STATE:
default:
break;
// These transitional states can remain after a decoding error. A retry (including
// entry disposal) must fail rather than return the same byte indefinitely.
throw new InvalidFormatException("BZip2: invalid decoder state");
}
return retChar;
}

View File

@@ -109,6 +109,9 @@ internal sealed class InflateBlocks
internal object checkfn; // check function
internal InflateCodes codes = new(); // if CODES, current state
internal int end; // one byte after sliding window
// Track initialized history separately from the circular read/write pointers.
internal int historyBytes;
internal int[] hufts; // single malloc for tree space
internal int index; // index into blens (or border)
internal InfTree inftree = new();
@@ -140,6 +143,7 @@ internal sealed class InflateBlocks
bitk = 0;
bitb = 0;
readAt = writeAt = 0;
historyBytes = 0;
if (checkfn != null)
{
@@ -148,6 +152,8 @@ internal sealed class InflateBlocks
return oldCheck;
}
internal void AddHistory(int count) => historyBytes = Math.Min(end, historyBytes + count);
internal int Process(int r)
{
int t; // temporary storage
@@ -340,6 +346,7 @@ internal sealed class InflateBlocks
t = m;
}
_codec.InputBuffer.AsSpan(p, t).CopyTo(window.Memory.Span.Slice(q));
AddHistory(t);
p += t;
n -= t;
q += t;
@@ -724,6 +731,7 @@ internal sealed class InflateBlocks
{
d.AsSpan(start, n).CopyTo(window.Memory.Span.Slice(0, n));
readAt = writeAt = n;
historyBytes = n;
}
// Returns true if inflate is currently at the end of a block generated
@@ -1174,15 +1182,29 @@ internal sealed class InflateCodes
b >>= j;
k -= j;
if (dist > blocks.historyBytes)
{
mode = BADCODE;
z.Message = "invalid distance too far back";
r = ZlibConstants.Z_DATA_ERROR;
blocks.bitb = b;
blocks.bitk = k;
z.AvailableBytesIn = n;
z.TotalBytesIn += p - z.NextIn;
z.NextIn = p;
blocks.writeAt = q;
return blocks.Flush(r);
}
mode = COPY;
goto case COPY;
case COPY: // o: copying bytes in window, waiting for space
f = q - dist;
while (f < 0)
if (f < 0)
{
// modulo window size-"while" instead
f += blocks.end; // of "if" handles invalid distances
f += blocks.end;
}
while (len != 0)
{
@@ -1220,6 +1242,7 @@ internal sealed class InflateCodes
}
blocks.window.Memory.Span[q++] = blocks.window.Memory.Span[f++];
blocks.AddHistory(1);
m--;
if (f == blocks.end)
@@ -1266,6 +1289,7 @@ internal sealed class InflateCodes
r = ZlibConstants.Z_OK;
blocks.window.Memory.Span[q++] = (byte)lit;
blocks.AddHistory(1);
m--;
mode = START;
@@ -1403,6 +1427,7 @@ internal sealed class InflateCodes
k -= (tp[tp_index_t_3 + 1]);
s.window.Memory.Span[q++] = (byte)tp[tp_index_t_3 + 2];
s.AddHistory(1);
m--;
continue;
}
@@ -1458,7 +1483,27 @@ internal sealed class InflateCodes
b >>= e;
k -= e;
if (d > s.historyBytes)
{
z.Message = "invalid distance too far back";
c = z.AvailableBytesIn - n;
c = (k >> 3) < c ? k >> 3 : c;
n += c;
p -= c;
k -= c << 3;
s.bitb = b;
s.bitk = k;
z.AvailableBytesIn = n;
z.TotalBytesIn += p - z.NextIn;
z.NextIn = p;
s.writeAt = q;
return ZlibConstants.Z_DATA_ERROR;
}
// do the copy
s.AddHistory(c);
m -= c;
if (q >= d)
{
@@ -1483,11 +1528,7 @@ internal sealed class InflateCodes
else
{
// else offset after destination
r = q - d;
do
{
r += s.end; // force pointer in window
} while (r < 0); // covers invalid distances
r = q - d + s.end;
e = s.end - r;
if (c > e)
{
@@ -1570,6 +1611,7 @@ internal sealed class InflateCodes
b >>= (tp[tp_index_t_3 + 1]);
k -= (tp[tp_index_t_3 + 1]);
s.window.Memory.Span[q++] = (byte)tp[tp_index_t_3 + 2];
s.AddHistory(1);
m--;
break;
}

View File

@@ -368,7 +368,9 @@ internal sealed class Lzma2EncoderStream : Stream
props[3] = (byte)(_dictionarySize >> 16);
props[4] = (byte)(_dictionarySize >> 24);
var decoder = _trialDecoder ??= new Decoder();
// This decoder verifies our own output using trusted encoder properties.
// An independent LZMA input limit must not disable LZMA2 compression.
var decoder = _trialDecoder ??= new Decoder(enforceDictionarySizeLimit: false);
decoder.SetDecoderProperties(props);
decoder.Code(input, Stream.Null, input.Length, uncompressedSize, null);

View File

@@ -250,6 +250,9 @@ internal static class Lzma2ParallelDecoder
{
ThrowHelper.ThrowIfNull(bufferPool);
// Reject folder-wide properties here, not inside workers where Parallel
// would wrap the format exception in AggregateException.
LzmaStream.GetLzma2DictionarySize(lzma2Props[0]);
Parallel.ForEach(
blocks,
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },

View File

@@ -5,6 +5,7 @@ using System.Buffers;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA.LZ;
using SharpCompress.Compressors.LZMA.RangeCoder;
@@ -252,8 +253,14 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
_rep2,
_rep3;
private readonly bool _enforceDictionarySizeLimit;
public Decoder()
: this(enforceDictionarySizeLimit: true) { }
internal Decoder(bool enforceDictionarySizeLimit)
{
_enforceDictionarySizeLimit = enforceDictionarySizeLimit;
_dictionarySize = -1;
}
@@ -266,9 +273,15 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
{
throw new InvalidParamException();
}
_outWindow = new OutWindow();
var blockSize = Math.Max(_dictionarySize, (1 << 12));
_outWindow.Create(blockSize);
if (_enforceDictionarySizeLimit)
{
DictionarySizeLimits.Validate(blockSize, DictionarySizeLimits.Lzma, "LZMA");
}
// Publish only a usable window so a rejected dictionary can be retried.
var window = new OutWindow();
window.Create(blockSize);
_outWindow = window;
}
private void SetLiteralProperties(int lp, int lc)

View File

@@ -4,6 +4,7 @@ using System.Buffers.Binary;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA.LZ;
using SharpCompress.IO;
@@ -65,6 +66,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
if (!isLzma2)
{
_dictionarySize = BinaryPrimitives.ReadInt32LittleEndian(properties.AsSpan(1));
DictionarySizeLimits.Validate(_dictionarySize, DictionarySizeLimits.Lzma, "LZMA");
_decoder = new Decoder();
_decoder.SetDecoderProperties(properties);
Properties = properties;
@@ -76,13 +78,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
else
{
// Larger standard dictionaries exceed this decoder's positive Int32 window range.
if (properties[0] > 37)
{
throw new InvalidParamException();
}
_dictionarySize = 2 | (properties[0] & 1);
_dictionarySize <<= (properties[0] >> 1) + 11;
_dictionarySize = GetLzma2DictionarySize(properties[0]);
Properties = new byte[1];
_availableBytes = 0;
@@ -90,6 +86,18 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
}
internal static int GetLzma2DictionarySize(byte property)
{
// Larger standard dictionaries exceed this decoder's positive Int32 window range.
if (property > 37)
{
throw new InvalidParamException();
}
var size = (2 | (property & 1)) << ((property >> 1) + 11);
DictionarySizeLimits.Validate(size, DictionarySizeLimits.Lzma2, "LZMA2");
return size;
}
public static LzmaStream Create(
byte[] properties,
Stream inputStream,

View File

@@ -3,6 +3,7 @@
using System;
using System.Buffers;
using System.Text;
using SharpCompress.Common;
namespace SharpCompress.Compressors.PPMd.H;
@@ -155,6 +156,7 @@ internal class SubAllocator : IDisposable
public virtual bool StartSubAllocator(int saSize)
{
DictionarySizeLimits.Validate(saSize, DictionarySizeLimits.Ppmd, "PPMd");
var t = saSize;
if (_subAllocatorSize == t)
{

View File

@@ -1,5 +1,7 @@
#nullable disable
using SharpCompress.Common;
namespace SharpCompress.Compressors.PPMd.I1;
/// Allocate a single, large array and then provide sections of this array to callers. Callers are provided with
@@ -128,6 +130,9 @@ internal class Allocator
/// <param name="allocatorSize"></param>
public void Start(int allocatorSize)
{
// PpmdProperties starts this allocator while parsing ZIP properties, so
// enforce the limit here, before any attacker-selected allocation.
DictionarySizeLimits.Validate(allocatorSize, DictionarySizeLimits.Ppmd, "PPMd");
var size = (uint)allocatorSize;
if (_allocatorSize != size)
{

View File

@@ -55,13 +55,14 @@ public class PpmdProperties
get => _allocatorSize;
set
{
_allocatorSize = value;
if (Version == PpmdVersion.I1)
{
_allocator ??= new Allocator();
_allocator.Start(_allocatorSize);
_allocator.Start(value);
}
// Keep the serialized properties in sync with the allocator if Start rejects the size.
_allocatorSize = value;
}
}

View File

@@ -130,6 +130,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack
private void Init()
{
DictionarySizeLimits.Validate(PackDef.MAXWINSIZE, DictionarySizeLimits.Rar4, "RAR4");
if (this.window is null)
{
this.window = ArrayPool<byte>.Shared.Rent(PackDef.MAXWINSIZE);

View File

@@ -111,14 +111,22 @@ internal sealed partial class Unpack : BitInput
WinSize = MinAllocSize;
}
// Validate before renting or reusing a solid window. The RAR5 header can
// request a huge dictionary even when there are no compressed bytes.
DictionarySizeLimits.Validate(
WinSize,
fileHeader.IsRar5 ? DictionarySizeLimits.Rar5 : DictionarySizeLimits.Rar4,
fileHeader.IsRar5 ? "RAR5" : "RAR4"
);
if (WinSize <= MaxWinSize) // Use the already allocated window.
{
return;
}
if ((WinSize >> 16) > 0x10000) // Window size must not exceed 4 GB.
if (WinSize > int.MaxValue)
{
return;
throw new InvalidFormatException("RAR window exceeds the supported array size.");
}
// Archiving code guarantees that window size does not grow in the same

View File

@@ -1,4 +1,5 @@
using System;
using SharpCompress.Common;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
@@ -10,6 +11,21 @@ public unsafe class Decompressor : IDisposable
public Decompressor()
{
handle = SafeDctxHandle.Create();
try
{
using var dctx = handle.Acquire();
Unsafe
.Methods.ZSTD_DCtx_setMaxWindowSize(
dctx,
checked((nuint)DictionarySizeLimits.ZStandard)
)
.EnsureZstdSuccess();
}
catch
{
handle.Dispose();
throw;
}
}
public void SetParameter(ZSTD_dParameter parameter, int value)

View File

@@ -3201,6 +3201,17 @@ public static unsafe partial class Methods
}
}
// Apply the limit before the single-pass shortcut too, so it
// does not depend on the caller's output buffer size.
zds->fParams.windowSize =
zds->fParams.windowSize > 1U << 10 ? zds->fParams.windowSize : 1U << 10;
if (zds->fParams.windowSize > zds->maxWindowSize)
{
return unchecked(
(nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge)
);
}
if (
zds->fParams.frameContentSize != unchecked(0UL - 1)
&& zds->fParams.frameType != ZSTD_frameType_e.ZSTD_skippableFrame
@@ -3282,15 +3293,6 @@ public static unsafe partial class Methods
zds->stage = ZSTD_dStage.ZSTDds_decodeBlockHeader;
}
zds->fParams.windowSize =
zds->fParams.windowSize > 1U << 10 ? zds->fParams.windowSize : 1U << 10;
if (zds->fParams.windowSize > zds->maxWindowSize)
{
return unchecked(
(nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge)
);
}
if (zds->maxBlockSizeParam != 0)
{
zds->fParams.blockSizeMax =

View File

@@ -631,6 +631,28 @@ public sealed class PooledMemoryStream : MemoryStream
}
}
// Metadata readers keep independent offsets over the same pooled storage.
internal byte ReadByteAt(int position)
{
EnsureNotClosed();
if ((uint)position >= (uint)_length)
{
throw new ArgumentOutOfRangeException(nameof(position));
}
return _blocks![position / _blockSize][position % _blockSize];
}
internal void ReadAt(int position, byte[] buffer, int offset, int count)
{
EnsureNotClosed();
ValidateReadWriteBufferArguments(buffer, offset, count);
if (position < 0 || position > _length || count > _length - position)
{
throw new ArgumentOutOfRangeException(nameof(position));
}
CopyFromSegmented(position, buffer, offset, count);
}
private void CopyFromSegmented(
int absoluteSourcePosition,
byte[] destination,

View File

@@ -2,7 +2,9 @@ using System;
using System.Buffers;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.BZip2;
using SharpCompress.Test.Mocks;
using Xunit;
@@ -11,6 +13,41 @@ namespace SharpCompress.Test.BZip2;
public class BZip2StreamAsyncTests
{
[Fact]
public async Task BZip2Stream_Throws_On_Read_After_Corrupt_Checksum_Async()
{
using var compressed = new MemoryStream();
await using (
var writer = await BZip2Stream.CreateAsync(
new AsyncOnlyStream(compressed),
SharpCompress.Compressors.CompressionMode.Compress,
false,
leaveOpen: true,
cancellationToken: CancellationToken.None
)
)
{
var data = Encoding.ASCII.GetBytes("BZip2 checksum validation test data.");
await writer.WriteAsync(data, 0, data.Length, CancellationToken.None);
}
var bytes = compressed.ToArray();
bytes[^5] ^= 1;
await using var stream = await BZip2Stream.CreateAsync(
new AsyncOnlyStream(new MemoryStream(bytes)),
SharpCompress.Compressors.CompressionMode.Decompress,
false,
cancellationToken: CancellationToken.None
);
await Assert.ThrowsAsync<ArchiveOperationException>(() =>
stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None)
);
await Assert.ThrowsAsync<InvalidFormatException>(() =>
stream.ReadAsync(new byte[1], 0, 1, CancellationToken.None)
);
}
private byte[] CreateTestData(int size)
{
var data = new byte[size];

View File

@@ -24,6 +24,8 @@ public class BZip2StreamTests
using var output = new MemoryStream();
Assert.Throws<ArchiveOperationException>(() => stream.CopyTo(output));
Assert.Throws<InvalidFormatException>(() => stream.ReadByte());
Assert.Throws<InvalidFormatException>(() => stream.Read(new byte[1], 0, 1));
}
// A stream that ends exactly where a block header is expected (no stream footer) is the shape a caller

View File

@@ -85,7 +85,8 @@ public class GZipArchiveAsyncTests : ArchiveTests
#endif
await using (var archive = await GZipArchive.OpenAsyncArchive(new AsyncOnlyStream(stream)))
{
await Assert.ThrowsAsync<NotSupportedException>(async () =>
// Shared cached entries avoid synchronous I/O on the async-only source.
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await archive.AddEntryAsync("jpg\\test.jpg", File.OpenRead(jpg), closeStream: true)
);
await archive.SaveToAsync(

View File

@@ -0,0 +1,173 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Archives.GZip;
using SharpCompress.Readers;
using SharpCompress.Test.Mocks;
using Xunit;
namespace SharpCompress.Test.GZip;
public class GZipArchiveCacheTests : TestBase
{
[Theory]
[InlineData(false, false, false)]
[InlineData(false, true, false)]
[InlineData(true, false, false)]
[InlineData(true, true, false)]
[InlineData(false, false, true)]
[InlineData(false, true, true)]
[InlineData(true, false, true)]
[InlineData(true, true, true)]
public async Task Partial_Enumeration_Can_Be_Resumed_Through_Either_Api(
bool asyncFirst,
bool asyncNext,
bool prefixed
)
{
using var input = new MemoryStream();
var start = prefixed ? 17 : 0;
input.Write(new byte[start], 0, start);
using (var fixture = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz")))
{
fixture.CopyTo(input);
}
input.Position = start;
using var archive = GZipArchive.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
var first = asyncFirst
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
var entries = asyncNext
? await asyncArchive.EntriesAsync.ToListAsync()
: archive.Entries.ToList();
Assert.Same(first, Assert.Single(entries));
Assert.Same(first, Assert.Single(archive.Entries));
Assert.Same(first, Assert.Single(await asyncArchive.EntriesAsync.ToListAsync()));
using var payload = first.OpenEntryStream();
Assert.True(payload.ReadByte() >= 0);
Assert.True(input.CanRead);
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task Completeness_Uses_The_Current_Entry_Cache(bool useAsync, bool partial)
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
using var archive = GZipArchive.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
if (partial)
{
if (useAsync)
{
await asyncArchive.EntriesAsync.FirstAsync();
}
else
{
_ = archive.Entries.First();
}
}
Assert.True(useAsync ? await asyncArchive.IsCompleteAsync() : archive.IsComplete);
var entry = Assert.Single(archive.Entries);
Assert.Same(entry, Assert.Single(await asyncArchive.EntriesAsync.ToListAsync()));
Assert.True(archive.IsComplete);
Assert.True(await asyncArchive.IsCompleteAsync());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Completeness_Does_Not_Reparse_Already_Cached_Entries(bool useAsync)
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
using var archive = GZipArchive.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
if (useAsync)
{
Assert.Single(await asyncArchive.EntriesAsync.ToListAsync());
}
else
{
Assert.Single(archive.Entries);
}
// Completeness needs only cached metadata and must not move the source.
input.Position = input.Length;
Assert.True(useAsync ? await asyncArchive.IsCompleteAsync() : archive.IsComplete);
Assert.Equal(input.Length, input.Position);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Extraction_Does_Not_Invalidate_A_Partially_Consumed_Loader(bool asyncFirst)
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
using var archive = GZipArchive.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
var first = asyncFirst
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
if (asyncFirst)
{
using var payload = await first.OpenEntryStreamAsync(CancellationToken.None);
await payload.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
else
{
using var payload = first.OpenEntryStream();
payload.CopyTo(Stream.Null);
}
var entries = asyncFirst
? archive.Entries.ToList()
: await asyncArchive.EntriesAsync.ToListAsync();
Assert.Same(first, Assert.Single(entries));
}
[Fact]
public async Task Resuming_And_Checking_Completeness_Do_Not_Use_Synchronous_Io()
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
using var asyncOnly = new AsyncOnlyStream(input, disposeStream: false);
await using (
var archive = await GZipArchive.OpenAsyncArchive(
asyncOnly,
ReaderOptions.ForExternalStream,
CancellationToken.None
)
)
{
var entry = await archive.EntriesAsync.FirstAsync();
Assert.True(await archive.IsCompleteAsync());
Assert.Same(entry, Assert.Single(await archive.EntriesAsync.ToListAsync()));
using var payload = await entry.OpenEntryStreamAsync(CancellationToken.None);
await payload.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
Assert.True(input.CanRead);
}
[Fact]
public async Task Cancelling_An_Enumeration_Does_Not_Poison_The_Shared_Cache()
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
await using var archive = await GZipArchive.OpenAsyncArchive(
input,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
using var cancellation = new CancellationTokenSource();
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
{
Assert.True(await entries.MoveNextAsync());
cancellation.Cancel();
await Assert.ThrowsAnyAsync<System.OperationCanceledException>(async () =>
await entries.MoveNextAsync()
);
}
Assert.Single(await archive.EntriesAsync.ToListAsync());
}
}

View File

@@ -0,0 +1,167 @@
using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Compressors.Xz;
using SharpCompress.Readers;
using SharpCompress.Test.Streams;
using Xunit;
namespace SharpCompress.Test.Rar;
[Collection(DictionarySizeLimitCollection.Name)]
public class RarDictionarySizeLimitTests
{
// The advisory's complete 72-byte archives have valid header CRCs, no packed
// data, and just 45,056 bytes of declared output. Neither needs a large window.
private const string Rar256MiB =
"526172211a070100b28180510c01050800070101c496868000c63c6d612a02030b8080000480e00220c809b1cf8059000c6578652f746573742e6578650a03020058a2ce32ffc101";
private const string Rar1GiB =
"526172211a070100b28180510c01050800070101c4968680000fd7ea442a02030b8080000480e00220c809b1cf8069000c6578652f746573742e6578650a03020058a2ce32ffc101";
public static IEnumerable<object[]> Payloads()
{
foreach (var hex in new[] { Rar256MiB, Rar1GiB })
{
foreach (var archiveApi in new[] { false, true })
{
foreach (var useAsync in new[] { false, true })
{
yield return new object[] { hex, archiveApi, useAsync };
}
}
}
}
[Theory]
[MemberData(nameof(Payloads))]
public async Task Rar5_Huge_Window_Is_Rejected_Before_Decoding(
string hex,
bool archiveApi,
bool useAsync
)
{
var bytes = DecodeHex(hex);
Assert.Equal(72, bytes.Length);
using var input = new MemoryStream(bytes);
var options = ReaderOptions.ForExternalStream;
InvalidFormatException exception;
if (archiveApi && useAsync)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
input,
options,
CancellationToken.None
);
var entry = await archive.EntriesAsync.FirstAsync();
Assert.Equal(45056, entry.Size);
Assert.Equal(0, entry.CompressedSize);
exception = await Assert.ThrowsAsync<InvalidFormatException>(async () =>
{
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
});
}
else if (archiveApi)
{
using var archive = ArchiveFactory.OpenArchive(input, options);
var entry = archive.Entries.First();
Assert.Equal(45056, entry.Size);
Assert.Equal(0, entry.CompressedSize);
exception = Assert.Throws<InvalidFormatException>(() =>
{
using var stream = entry.OpenEntryStream();
});
}
else if (useAsync)
{
using var forwardOnly = new NonSeekableStream(input);
await using var reader = await ReaderFactory.OpenAsyncReader(
forwardOnly,
options,
CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
exception = await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None)
);
}
else
{
using var forwardOnly = new NonSeekableStream(input);
using var reader = ReaderFactory.OpenReader(forwardOnly, options);
Assert.True(reader.MoveToNextEntry());
exception = Assert.Throws<InvalidFormatException>(() =>
reader.WriteEntryTo(Stream.Null)
);
}
Assert.Contains("RAR5 dictionary size", exception.Message);
Assert.Contains(DictionarySizeLimits.Rar5.ToString(), exception.Message);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Rar5_Custom_Limit_Accounts_For_Minimum_Window_Allocation(bool useAsync)
{
using var limits = new DictionarySizeLimitScope();
var bytes = DecodeHex(Rar256MiB);
// Change the compression-info vint from 0x2c80 to 0x0080: a 128 KiB
// declared dictionary. The unpacker requires at least a 256 KiB window.
bytes[46] = 1;
var crc = ~Crc32.Update(Crc32.DefaultSeed, bytes.AsSpan(29));
BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(25), crc);
foreach (var maximum in new[] { 128L << 10, 256L << 10 })
{
using var input = new MemoryStream(bytes);
DictionarySizeLimits.Rar5 = maximum;
var options = ReaderOptions.ForExternalStream;
if (useAsync)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
input,
options,
CancellationToken.None
);
var entry = await archive.EntriesAsync.FirstAsync();
if (maximum < 256L << 10)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
{
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
});
}
else
{
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
}
}
else
{
using var archive = ArchiveFactory.OpenArchive(input, options);
var entry = archive.Entries.First();
if (maximum < 256L << 10)
{
Assert.Throws<InvalidFormatException>(() =>
{
using var stream = entry.OpenEntryStream();
});
}
else
{
using var stream = entry.OpenEntryStream();
}
}
}
}
private static byte[] DecodeHex(string hex) =>
Enumerable
.Range(0, hex.Length / 2)
.Select(i => Convert.ToByte(hex.Substring(i * 2, 2), 16))
.ToArray();
}

View File

@@ -0,0 +1,67 @@
using System;
using System.IO;
using System.Text;
using SharpCompress.Common;
using SharpCompress.Common.SevenZip;
using SharpCompress.Crypto;
using SharpCompress.IO;
using Xunit;
namespace SharpCompress.Test.SevenZip;
public class SevenZipDataReaderTests
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public void DataReader_Reads_And_Clones_Across_Storage_Boundaries(bool pooled)
{
using var data = new MemoryStream();
using (var writer = new BinaryWriter(data, Encoding.UTF8, true))
{
writer.Write(0x12345678u);
writer.Write(0x123456789abcdef0ul);
}
data.WriteEncodedUInt64(ulong.MaxValue);
// The surrogate pair crosses a UTF-16 decoding chunk as well as pooled blocks.
var name = new string('x', 127) + "\U0001f600" + new string('y', 130);
var nameBytes = Encoding.Unicode.GetBytes(name + "\0");
data.Write(nameBytes, 0, nameBytes.Length);
using var storage = new PooledMemoryStream(0, 7);
var bytes = data.ToArray();
storage.Write(bytes, 0, bytes.Length);
var reader = pooled ? new DataReader(storage) : new DataReader(bytes, 0, bytes.Length);
Assert.Equal(0x12345678u, reader.ReadUInt32());
var clone = reader.Clone();
Assert.Equal(0x123456789abcdef0ul, clone.ReadUInt64());
Assert.Equal(4, reader.Offset);
Assert.Equal(0x123456789abcdef0ul, reader.ReadUInt64());
Assert.Equal(ulong.MaxValue, reader.ReadNumber());
Assert.Equal(ulong.MaxValue, clone.ReadNumber());
Assert.Equal(name, reader.ReadString());
Assert.Equal(name, clone.ReadString());
Assert.Equal(0, reader.Remaining);
Assert.Throws<IncompleteArchiveException>(() => reader.ReadByte());
Assert.Throws<IncompleteArchiveException>(() => reader.SkipData(1));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void DataReader_Bounds_Bulk_Reads_And_Unterminated_Strings(bool pooled)
{
byte[] bytes = [1, 2, 3, 4, 5];
using var storage = new PooledMemoryStream(0, 3);
storage.Write(bytes, 0, bytes.Length);
var reader = pooled ? new DataReader(storage) : new DataReader(bytes, 0, bytes.Length);
var output = new byte[4];
reader.SkipData(1);
reader.ReadBytes(output, 0, output.Length);
Assert.Equal(new byte[] { 2, 3, 4, 5 }, output);
Assert.Throws<IncompleteArchiveException>(() => reader.ReadBytes(output, 0, 1));
var stringReader = pooled
? new DataReader(storage)
: new DataReader(bytes, 0, bytes.Length);
Assert.Throws<IncompleteArchiveException>(() => stringReader.ReadString());
}
}

View File

@@ -0,0 +1,122 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
using Xunit;
namespace SharpCompress.Test.SevenZip;
public class SevenZipDeflateSolidTests : TestBase
{
private static readonly int[] Sizes = [1000, 70000, 100000, 200000, 500000, 1048576, 3000000];
// Generated with libarchive bsdtar --format 7zip --options compression=deflate.
// Each file repeats a 4096-byte xorshift sequence with a different starting offset.
private static byte[] CreateExpected(int index)
{
var block = new byte[4096];
uint state = 0x12345678;
for (var i = 0; i < block.Length; i++)
{
state ^= state << 13;
state ^= state >> 17;
state ^= state << 5;
block[i] = (byte)state;
}
var data = new byte[Sizes[index]];
for (var i = 0; i < data.Length; i++)
{
data[i] = block[(i + index * 997) % block.Length];
}
return data;
}
[Theory]
[InlineData(512, 0)]
[InlineData(4096, 1)]
[InlineData(65536, 2)]
[InlineData(131072, 0)]
[InlineData(300000, 0)]
public void SevenZipArchive_Solid_Deflate_ExtractAllEntries(int bufferSize, int firstEntryMode)
{
using var archive = SevenZipArchive.OpenArchive(
Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.Deflate.7z")
);
Assert.True(archive.IsSolid);
using var reader = archive.ExtractAllEntries();
var index = 0;
while (reader.MoveToNextEntry())
{
Assert.Equal($"{index}.bin", reader.Entry.Key);
Assert.Equal(Sizes[index], reader.Entry.Size);
if (index == 0 && firstEntryMode == 2)
{
index++;
continue;
}
using var stream = reader.OpenEntryStream();
if (index == 0 && firstEntryMode == 1)
{
Assert.True(stream.Read(new byte[17], 0, 17) > 0);
}
else
{
using var output = new MemoryStream();
stream.CopyTo(output, bufferSize);
Assert.Equal(CreateExpected(index), output.ToArray());
}
index++;
}
Assert.Equal(Sizes.Length, index);
}
[Theory]
[InlineData(512, 0)]
[InlineData(4096, 1)]
[InlineData(65536, 2)]
[InlineData(131072, 0)]
[InlineData(300000, 0)]
public async Task SevenZipArchive_Solid_Deflate_ExtractAllEntries_Async(
int bufferSize,
int firstEntryMode
)
{
await using var archive = await SevenZipArchive.OpenAsyncArchive(
Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.Deflate.7z"),
cancellationToken: CancellationToken.None
);
Assert.True(await archive.IsSolidAsync());
await using var reader = await archive.ExtractAllEntriesAsync();
var index = 0;
while (await reader.MoveToNextEntryAsync(CancellationToken.None))
{
Assert.Equal($"{index}.bin", reader.Entry.Key);
Assert.Equal(Sizes[index], reader.Entry.Size);
if (index == 0 && firstEntryMode == 2)
{
index++;
continue;
}
await using var stream = await reader.OpenEntryStreamAsync(CancellationToken.None);
if (index == 0 && firstEntryMode == 1)
{
Assert.True(
await stream.ReadAsync(new byte[17], 0, 17, CancellationToken.None) > 0
);
}
else
{
using var output = new MemoryStream();
await stream.CopyToAsync(output, bufferSize, CancellationToken.None);
Assert.Equal(CreateExpected(index), output.ToArray());
}
index++;
}
Assert.Equal(Sizes.Length, index);
}
}

View File

@@ -0,0 +1,501 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Common.SevenZip;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Crypto;
using SharpCompress.Test.Mocks;
using Xunit;
namespace SharpCompress.Test.SevenZip;
[CollectionDefinition(nameof(SevenZipHeaderAllocationTests), DisableParallelization = true)]
public class SevenZipHeaderAllocationCollection { }
[Collection(nameof(SevenZipHeaderAllocationTests))]
public class SevenZipHeaderAllocationTests
{
public static IEnumerable<object[]> MalformedHeaders()
{
var count = Number(100_000);
yield return ["folders", Join([1, 4, 7, 11], count, [0])];
yield return ["files", Join([1, 5], count, [0])];
yield return ["coders", Join([1, 4, 7, 11, 1, 0], count)];
yield return ["properties", Join([1, 4, 7, 11, 1, 0, 1, 0x21, 0x21], count)];
yield return ["pack streams", Join([1, 4, 6, 0], count, [9, 0])];
yield return ["input streams", Join([1, 4, 7, 11, 1, 0, 1, 0x11, 0], count, [1])];
yield return ["output streams", Join([1, 4, 7, 11, 1, 0, 1, 0x11, 0, 1], count)];
yield return ["zero coders", new byte[] { 1, 4, 7, 11, 1, 0, 0 }];
yield return ["zero input streams", new byte[] { 1, 4, 7, 11, 1, 0, 1, 0x11, 0, 0, 1 }];
yield return ["zero output streams", new byte[] { 1, 4, 7, 11, 1, 0, 1, 0x11, 0, 1, 0 }];
yield return ["truncated empty bitmap", Join([1, 5], count, [14, 1, 0x80, 0])];
// One valid Copy folder followed by forged per-folder substream counts.
byte[] folder = [1, 4, 7, 11, 1, 0, 1, 1, 0, 12, 0, 0, 8, 13];
yield return ["substream CRCs", Join(folder, count, [10, 1])];
yield return ["substream sizes", Join(folder, count, [9, 0])];
yield return ["implicit substream CRCs", Join(folder, count, [0])];
yield return ["substream size exceeds folder", Join(folder, [2, 9, 1, 0])];
yield return ["invalid bind index", new byte[] { 1, 4, 7, 11, 1, 0, 2, 1, 0, 1, 0, 2, 0 }];
yield return
[
"invalid pack index",
new byte[] { 1, 4, 7, 11, 1, 0, 1, 0x11, 0, 2, 1, 0, 2 },
];
yield return
[
"duplicate pack index",
new byte[] { 1, 4, 7, 11, 1, 0, 1, 0x11, 0, 2, 1, 0, 0 },
];
yield return
[
"oversized decoded metadata",
MetadataHeader(true, (ulong)int.MaxValue + 1, 0, [1, 0]),
];
yield return
[
"overflowing inputs",
Join(
[1, 4, 7, 11, 1, 0, 2, 0x11, 0],
Number(int.MaxValue),
[1, 0x11, 0],
Number(int.MaxValue),
[1]
),
];
}
[Theory]
[MemberData(nameof(MalformedHeaders))]
public void SevenZipArchive_InvalidHeaderCounts_RejectBeforeAllocation(
string name,
byte[] header
)
{
using var stream = CreateArchive(header);
using var archive = ArchiveFactory.OpenArchive(stream);
#if !LEGACY_DOTNET
var before = GC.GetAllocatedBytesForCurrentThread();
#endif
var exception = Record.Exception(() => archive.Entries.ToList());
Assert.True(
exception is InvalidFormatException or IncompleteArchiveException,
$"{name}: {exception}"
);
#if !LEGACY_DOTNET
var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
Assert.True(allocated < 512 * 1024, $"{name}: allocated {allocated} bytes.");
#endif
Assert.True(stream.CanRead);
}
[Theory]
[MemberData(nameof(MalformedHeaders))]
public async Task SevenZipArchive_InvalidHeaderCounts_AsyncRejectBeforeAllocation(
string name,
byte[] header
)
{
using var stream = CreateArchive(header);
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
#if !LEGACY_DOTNET
var before = GC.GetTotalAllocatedBytes(precise: true);
#endif
var exception = await Record.ExceptionAsync(async () =>
{
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
) { }
});
Assert.True(
exception is InvalidFormatException or IncompleteArchiveException,
$"{name}: {exception}"
);
#if !LEGACY_DOTNET
var allocated = GC.GetTotalAllocatedBytes(precise: true) - before;
Assert.True(allocated < 512 * 1024, $"{name}: allocated {allocated} bytes.");
#endif
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void SevenZipArchive_TruncatedDecodedMetadata_DoesNotReserveUnpackSize(
bool encodedHeader
)
{
// Copy produces only the bytes present, despite advertising 16 MiB.
var header = MetadataHeader(encodedHeader, 16 * 1024 * 1024, 1, [1, 0]);
using var stream = CreateArchive(header, [0]);
using var archive = ArchiveFactory.OpenArchive(stream);
#if !LEGACY_DOTNET
var before = GC.GetAllocatedBytesForCurrentThread();
#endif
Assert.Throws<IncompleteArchiveException>(() => archive.Entries.ToList());
#if !LEGACY_DOTNET
var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
Assert.True(allocated < 512 * 1024, $"Allocated {allocated} bytes.");
#endif
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_TruncatedDecodedMetadata_AsyncRejects(bool encodedHeader)
{
var header = MetadataHeader(encodedHeader, 16 * 1024 * 1024, 1, [1, 0]);
using var stream = CreateArchive(header, [0]);
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
#if !LEGACY_DOTNET
var before = GC.GetTotalAllocatedBytes(precise: true);
#endif
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
{
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
) { }
});
#if !LEGACY_DOTNET
var allocated = GC.GetTotalAllocatedBytes(precise: true) - before;
Assert.True(allocated < 512 * 1024, $"Allocated {allocated} bytes.");
#endif
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_ReportedLzma2MetadataPayload_Rejects(bool async)
{
// The report puts the first packed byte at the start of the next header.
var header = MetadataHeader(false, 16 * 1024 * 1024, 1, [0x21, 0x21, 1, 0x18]);
using var stream = CreateArchive(header);
if (async)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
#if !LEGACY_DOTNET
var before = GC.GetTotalAllocatedBytes(precise: true);
#endif
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
{
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
) { }
});
#if !LEGACY_DOTNET
var allocated = GC.GetTotalAllocatedBytes(precise: true) - before;
Assert.True(allocated < 512 * 1024, $"Allocated {allocated} bytes.");
#endif
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
#if !LEGACY_DOTNET
var before = GC.GetAllocatedBytesForCurrentThread();
#endif
Assert.Throws<DataErrorException>(() => archive.Entries.ToList());
#if !LEGACY_DOTNET
var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
Assert.True(allocated < 512 * 1024, $"Allocated {allocated} bytes.");
#endif
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_BitPackedEmptyFiles_AreValid(bool async)
{
// Nine nameless directories need only two bitmap bytes. A byte-per-file
// bound on the remaining header would incorrectly reject this archive.
using var stream = CreateArchive([1, 5, 9, 14, 2, 0xFF, 0x80, 0, 0]);
if (async)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
stream,
cancellationToken: CancellationToken.None
);
var count = 0;
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
)
{
Assert.True(entry.IsDirectory);
count++;
}
Assert.Equal(9, count);
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
Assert.Equal(9, archive.Entries.Count());
Assert.All(archive.Entries, entry => Assert.True(entry.IsDirectory));
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_ExternalFolderCount_UsesExternalBuffer(bool async)
{
// Decode one byte of additional metadata, then reference it as the
// folder-definition buffer. The inline header has ample padding.
var header = Join(MetadataHeader(false, 1, 1, [1, 0]), [4, 7, 11, 2, 1, 0], new byte[128]);
using var stream = CreateArchive(header, [0]);
if (async)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
{
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
) { }
});
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
Assert.Throws<IncompleteArchiveException>(() => archive.Entries.ToList());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_ExternalFolderDefinition_IsValid(bool async)
{
// Additional metadata contains a Copy folder. Its unpack size remains
// inline; the file's packed byte follows the external folder definition.
var header = Join(
MetadataHeader(false, 3, 3, [1, 0]),
[4, 6, 3, 1, 9, 1, 0, 7, 11, 1, 1, 0, 12, 1, 0, 8, 0, 0, 5, 1, 0, 0]
);
using var stream = CreateArchive(header, [1, 1, 0, 42]);
if (async)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
var count = 0;
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
)
{
using var entryStream = await entry.OpenEntryStreamAsync(CancellationToken.None);
var data = new byte[1];
Assert.Equal(1, await entryStream.ReadAsync(data, 0, 1, CancellationToken.None));
Assert.Equal(42, data[0]);
count++;
}
Assert.Equal(1, count);
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
using var entryStream = archive.Entries.Single().OpenEntryStream();
Assert.Equal(42, entryStream.ReadByte());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_DecodedMetadataSpanningBuffers_IsValid(bool async)
{
// An unknown archive property forces decoding across several buffers.
var decoded = Join([1, 2, 26], Number(20_000), new byte[20_000], [0, 0]);
var header = MetadataHeader(true, (ulong)decoded.Length, (ulong)decoded.Length, [1, 0]);
using var stream = CreateArchive(header, decoded);
if (async)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
await foreach (
var entry in archive.EntriesAsync.WithCancellation(CancellationToken.None)
)
{
Assert.Fail("Expected an empty archive.");
}
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
Assert.Empty(archive.Entries);
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(2, 0, false)]
[InlineData(2, 0, true)]
[InlineData(2, 127, false)]
[InlineData(2, 127, true)]
[InlineData(3, 0, false)]
[InlineData(3, 0, true)]
public async Task SevenZipArchive_Property_Size_Validation_Is_Version_Aware(
byte minorVersion,
byte declaredSize,
bool useAsync
)
{
// Both known properties have deliberately inaccurate sizes, including
// a name preceding the empty-stream bitmap.
using var stream = CreateArchive([
1,
5,
1,
17,
declaredSize,
0,
(byte)'x',
0,
0,
0,
14,
declaredSize,
0x80,
0,
0,
]);
stream.GetBuffer()[7] = minorVersion;
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
cancellationToken: CancellationToken.None
);
if (minorVersion > 2)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await archive.EntriesAsync.ToListAsync()
);
}
else
{
var entry = Assert.Single(await archive.EntriesAsync.ToListAsync());
Assert.True(entry.IsDirectory);
Assert.Equal("x", entry.Key);
}
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
if (minorVersion > 2)
{
Assert.ThrowsAny<SharpCompressException>(() => archive.Entries.ToList());
}
else
{
var entry = Assert.Single(archive.Entries);
Assert.True(entry.IsDirectory);
Assert.Equal("x", entry.Key);
}
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZipArchive_Legacy_File_Count_Is_Bounded_By_Available_Metadata(
bool useAsync
)
{
using var stream = CreateArchive(Join([1, 5], Number(100_000), [14, 0, 0x80, 0]));
stream.GetBuffer()[7] = 2;
if (useAsync)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
stream,
cancellationToken: CancellationToken.None
);
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
await archive.EntriesAsync.ToListAsync()
);
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream);
Assert.Throws<IncompleteArchiveException>(() => archive.Entries.ToList());
}
}
private static byte[] MetadataHeader(
bool encoded,
ulong unpackSize,
ulong packSize,
byte[] coder
) =>
Join(
encoded ? [23] : [1, 3],
[6, 0, 1, 9],
Number(packSize),
[0, 7, 11, 1, 0, 1],
coder,
[12],
Number(unpackSize),
[0, 0]
);
private static byte[] Number(ulong value)
{
using var stream = new MemoryStream();
stream.WriteEncodedUInt64(value);
return stream.ToArray();
}
private static byte[] Join(params byte[][] parts) => parts.SelectMany(part => part).ToArray();
private static MemoryStream CreateArchive(byte[] header, byte[]? packedData = null)
{
packedData ??= [];
var crc = Crc32Stream.Compute(
Crc32Stream.DEFAULT_POLYNOMIAL,
Crc32Stream.DEFAULT_SEED,
header
);
var stream = new MemoryStream();
SevenZipSignatureHeaderWriter.WriteFinal(
stream,
(ulong)packedData.Length,
(ulong)header.Length,
crc
);
stream.Write(packedData, 0, packedData.Length);
stream.Write(header, 0, header.Length);
stream.Position = 0;
return stream;
}
}

View File

@@ -11,6 +11,7 @@ using Xunit;
namespace SharpCompress.Test.SevenZip;
[Collection(nameof(SevenZipHeaderAllocationTests))]
public class SevenZipHeaderLifetimeTests : TestBase
{
[Theory]
@@ -113,14 +114,125 @@ public class SevenZipHeaderLifetimeTests : TestBase
Assert.True(input.CanRead);
}
[Theory]
[InlineData(false, 0)]
[InlineData(true, 0)]
[InlineData(false, 1)]
[InlineData(true, 1)]
[InlineData(false, 2)]
[InlineData(true, 2)]
[InlineData(true, 3)]
public async Task Decoded_Metadata_Is_Not_Copied_And_Returns_All_Buffers(
bool asyncRead,
int failure
)
{
using var decoded = new MemoryStream();
decoded.Write(new byte[] { failure == 1 ? (byte)99 : (byte)1, 2, 26 }, 0, 3);
decoded.WriteEncodedUInt64(1 << 20);
var padding = new byte[1 << 20];
decoded.Write(padding, 0, padding.Length);
decoded.WriteByte(0);
decoded.WriteByte(0);
using var encoded = new MemoryStream();
encoded.Write(new byte[] { 23, 6, 0, 1, 9 }, 0, 5);
encoded.WriteEncodedUInt64((ulong)decoded.Length);
encoded.Write(new byte[] { 0, 7, 11, 1, 0, 1, 1, 0, 12 }, 0, 9);
encoded.WriteEncodedUInt64((ulong)decoded.Length + (failure == 2 ? 8192ul : 0));
encoded.WriteByte(0);
encoded.WriteByte(0);
var header = encoded.ToArray();
using var input = new MemoryStream();
SevenZipSignatureHeaderWriter.WriteFinal(
input,
(ulong)decoded.Length,
(ulong)header.Length,
Crc32Stream.Compute(header)
);
decoded.Position = 0;
decoded.CopyTo(input);
input.Write(header, 0, header.Length);
input.Position = 0;
using var cancellation = new CancellationTokenSource();
var pool = new TrackingPool
{
OnRent =
failure == 3
? count =>
{
// Scratch and packed-input buffers are rented before metadata storage.
if (count == 3)
{
cancellation.Cancel();
}
}
: null,
};
var reader = new ArchiveReader(pool);
if (asyncRead)
{
await reader.OpenAsync(input, false, CancellationToken.None);
}
else
{
reader.Open(input, false);
}
#if !LEGACY_DOTNET
var before = GC.GetTotalAllocatedBytes(precise: true);
#endif
if (failure == 3)
{
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
await reader.ReadDatabaseAsync(new NoPassword(), cancellation.Token)
);
}
else if (failure == 0)
{
var database = asyncRead
? await reader.ReadDatabaseAsync(new NoPassword(), CancellationToken.None)
: reader.ReadDatabase(new NoPassword());
Assert.Empty(database._files);
}
else if (asyncRead)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await reader.ReadDatabaseAsync(new NoPassword(), CancellationToken.None)
);
}
else
{
Assert.ThrowsAny<SharpCompressException>(() => reader.ReadDatabase(new NoPassword()));
}
#if !LEGACY_DOTNET
var allocated = GC.GetTotalAllocatedBytes(precise: true) - before;
Assert.True(
allocated < pool.AllocatedBytes + 512 * 1024,
$"Allocated {allocated} bytes including {pool.AllocatedBytes} bytes of pooled storage."
);
#endif
Assert.True(pool.Rents > 0);
Assert.Equal(1, pool.ScratchRents);
Assert.Equal(pool.Rents, pool.Returns);
Assert.True(input.CanRead);
}
private sealed class TrackingPool : ArrayPool<byte>
{
public int Rents { get; private set; }
public int Returns { get; private set; }
public int ScratchRents { get; private set; }
public long AllocatedBytes { get; private set; }
public Action<int>? OnRent { get; set; }
public override byte[] Rent(int minimumLength)
{
Rents++;
if (minimumLength == 8192)
{
ScratchRents++;
}
AllocatedBytes += minimumLength;
OnRent?.Invoke(Rents);
return new byte[minimumLength];
}

View File

@@ -0,0 +1,292 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors;
using SharpCompress.Compressors.Deflate;
using Xunit;
namespace SharpCompress.Test.Streams;
public class DeflateHistoryTests
{
[Theory]
[InlineData(false, false, false)]
[InlineData(false, false, true)]
[InlineData(false, true, false)]
[InlineData(false, true, true)]
[InlineData(true, false, false)]
[InlineData(true, false, true)]
[InlineData(true, true, false)]
[InlineData(true, true, true)]
public async Task Deflate_Rejects_Distance_Beyond_Output_History(
bool useAsync,
bool useFastPath,
bool emptyHistory
)
{
// Fixed Huffman blocks: either a match before any literal, or 'A' then length 3/distance 4.
byte[] compressed = emptyHistory ? [0x03, 0x02, 0x00] : [0x73, 0x04, 0x62, 0x27, 0x00];
using var input = new MemoryStream(WithReadAhead(compressed, useFastPath));
using var decoder = new DeflateStream(input, CompressionMode.Decompress);
using var output = new MemoryStream();
ZlibException exception;
if (useAsync)
{
exception = await Assert.ThrowsAsync<ZlibException>(() =>
decoder.CopyToAsync(output, 1024, CancellationToken.None)
);
}
else
{
exception = Assert.Throws<ZlibException>(() => decoder.CopyTo(output));
}
Assert.Contains("invalid distance too far back", exception.Message);
Assert.Empty(output.ToArray());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Deflate_Invalid_Distance_Does_Not_Copy_Uninitialized_History(bool useFastPath)
{
var codec = new ZlibCodec();
codec.InitializeInflate(false);
try
{
// Seed this decoder's window directly, without relying on shared-pool reuse.
codec.istate.blocks.window.Memory.Span.Fill(0xa5);
SetInput(codec, WithReadAhead([0x73, 0x04, 0x62, 0x27, 0x00], useFastPath));
codec.OutputBuffer = new byte[16];
codec.AvailableBytesOut = codec.OutputBuffer.Length;
Assert.Throws<ZlibException>(() => codec.Inflate(FlushType.None));
Assert.Equal(1, codec.NextOut);
Assert.Equal((byte)'A', codec.OutputBuffer[0]);
Assert.Equal(new byte[15], codec.OutputBuffer.AsSpan(1).ToArray());
}
finally
{
codec.EndInflate();
}
}
[Theory]
[InlineData(false, 3)]
[InlineData(false, 4)]
[InlineData(true, 3)]
[InlineData(true, 4)]
public void Deflate_Preset_Dictionary_Limits_Available_History(
bool useFastPath,
int dictionarySize
)
{
var codec = new ZlibCodec();
codec.InitializeInflate(false);
try
{
byte[] dictionary = [(byte)'A', (byte)'B', (byte)'C', (byte)'D'];
codec.istate.blocks.SetDictionary(dictionary, 0, dictionarySize);
// Length 3/distance 4, with no preceding literal.
SetInput(codec, WithReadAhead([0x03, 0x62, 0x00], useFastPath));
codec.OutputBuffer = new byte[16];
codec.AvailableBytesOut = codec.OutputBuffer.Length;
if (dictionarySize < 4)
{
var exception = Assert.Throws<ZlibException>(() => codec.Inflate(FlushType.None));
Assert.Contains("invalid distance too far back", exception.Message);
Assert.Equal(0, codec.NextOut);
}
else
{
Assert.Equal(ZlibConstants.Z_STREAM_END, codec.Inflate(FlushType.None));
Assert.Equal(3, codec.NextOut);
Assert.Equal(
dictionary.AsSpan(0, 3).ToArray(),
codec.OutputBuffer.AsSpan(0, 3).ToArray()
);
}
}
finally
{
codec.EndInflate();
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Deflate_Reset_Discards_Previous_History(bool useFastPath)
{
var codec = new ZlibCodec();
codec.InitializeInflate(false);
try
{
codec.istate.blocks.SetDictionary([1, 2, 3, 4], 0, 4);
codec.istate.Reset();
SetInput(codec, WithReadAhead([0x03, 0x62, 0x00], useFastPath));
codec.OutputBuffer = new byte[16];
codec.AvailableBytesOut = codec.OutputBuffer.Length;
var exception = Assert.Throws<ZlibException>(() => codec.Inflate(FlushType.None));
Assert.Contains("invalid distance too far back", exception.Message);
Assert.Equal(0, codec.NextOut);
}
finally
{
codec.EndInflate();
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Deflate_Rejects_Distance_Beyond_Window_Size(bool useFastPath)
{
var codec = new ZlibCodec();
codec.InitializeInflate(9, false);
try
{
// Produce more than the 512-byte window, then request length 3/distance 513.
using var compressed = new MemoryStream();
WriteStoredBlock(compressed, new byte[513]);
var match = WithReadAhead([0x03, 0x26, 0x00, 0x00], useFastPath);
compressed.Write(match, 0, match.Length);
SetInput(codec, compressed.ToArray());
codec.OutputBuffer = new byte[1024];
codec.AvailableBytesOut = codec.OutputBuffer.Length;
var exception = Assert.Throws<ZlibException>(() => codec.Inflate(FlushType.None));
Assert.Contains("invalid distance too far back", exception.Message);
Assert.Equal(513, codec.NextOut);
}
finally
{
codec.EndInflate();
}
}
[Theory]
[InlineData(false, 3)]
[InlineData(true, 3)]
[InlineData(false, 32768)]
[InlineData(true, 32768)]
[InlineData(false, 32769)]
[InlineData(true, 32769)]
public async Task Deflate_History_Survives_Stored_Blocks_And_Window_Wrap(
bool useAsync,
int historySize
)
{
var history = new byte[historySize];
for (var i = 0; i < history.Length; i++)
{
history[i] = (byte)(i % 251);
}
// Follow a non-final stored block with a fixed block referencing the available history.
var distance = Math.Min(historySize, 32768);
byte[] match = distance == 3 ? [0x03, 0x22, 0x00] : [0x03, 0xde, 0xff, 0x0f, 0x00];
using var compressed = new MemoryStream();
WriteStoredBlock(compressed, history);
var tail = WithReadAhead(match, true);
compressed.Write(tail, 0, tail.Length);
compressed.Position = 0;
using var decoder = new DeflateStream(compressed, CompressionMode.Decompress);
using var output = new MemoryStream();
if (useAsync)
{
await decoder.CopyToAsync(output, 137, CancellationToken.None);
}
else
{
decoder.CopyTo(output, 137);
}
var expected = new byte[history.Length + 3];
history.CopyTo(expected, 0);
Array.Copy(history, history.Length - distance, expected, history.Length, 3);
Assert.Equal(expected, output.ToArray());
}
[Theory]
[InlineData(false, 1)]
[InlineData(true, 1)]
[InlineData(false, 137)]
[InlineData(true, 137)]
public async Task Deflate_History_Survives_Overlapping_And_Partial_Matches(
bool useAsync,
int outputBufferSize
)
{
var expected = new byte[70000];
for (var i = 0; i < expected.Length; i++)
{
expected[i] = (byte)(i % 7);
}
using var compressed = new MemoryStream();
using (
var encoder = new System.IO.Compression.DeflateStream(
compressed,
System.IO.Compression.CompressionMode.Compress,
leaveOpen: true
)
)
{
encoder.Write(expected, 0, expected.Length);
}
compressed.Position = 0;
using var decoder = new DeflateStream(compressed, CompressionMode.Decompress);
using var output = new MemoryStream();
var buffer = new byte[outputBufferSize];
int read;
while (
(
read = useAsync
? await decoder.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None)
: decoder.Read(buffer, 0, buffer.Length)
) != 0
)
{
output.Write(buffer, 0, read);
}
Assert.Equal(expected, output.ToArray());
}
private static byte[] WithReadAhead(byte[] compressed, bool useFastPath)
{
if (!useFastPath)
{
return compressed;
}
// InflateFast requires at least ten input bytes; bytes after the final block are read-ahead.
var padded = new byte[compressed.Length + 16];
compressed.CopyTo(padded, 0);
return padded;
}
private static void WriteStoredBlock(Stream compressed, byte[] history)
{
compressed.WriteByte(0);
compressed.WriteByte((byte)history.Length);
compressed.WriteByte((byte)(history.Length >> 8));
compressed.WriteByte((byte)~history.Length);
compressed.WriteByte((byte)(~history.Length >> 8));
compressed.Write(history, 0, history.Length);
}
private static void SetInput(ZlibCodec codec, byte[] compressed)
{
codec.InputBuffer = compressed;
codec.NextIn = 0;
codec.AvailableBytesIn = compressed.Length;
}
}

View File

@@ -0,0 +1,565 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Compressors;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.PPMd;
using SharpCompress.Compressors.PPMd.H;
using SharpCompress.Compressors.ZStandard;
using SharpCompress.Compressors.ZStandard.Unsafe;
using SharpCompress.Readers;
using Xunit;
namespace SharpCompress.Test.Streams;
// Static settings must not change while other collections are extracting archives.
[CollectionDefinition(Name, DisableParallelization = true)]
public class DictionarySizeLimitCollection
{
public const string Name = "Dictionary size limits";
}
internal sealed class DictionarySizeLimitScope : IDisposable
{
private readonly long rar4 = DictionarySizeLimits.Rar4;
private readonly long rar5 = DictionarySizeLimits.Rar5;
private readonly long lzma = DictionarySizeLimits.Lzma;
private readonly long lzma2 = DictionarySizeLimits.Lzma2;
private readonly long ppmd = DictionarySizeLimits.Ppmd;
private readonly long zStandard = DictionarySizeLimits.ZStandard;
public void Dispose()
{
DictionarySizeLimits.Rar4 = rar4;
DictionarySizeLimits.Rar5 = rar5;
DictionarySizeLimits.Lzma = lzma;
DictionarySizeLimits.Lzma2 = lzma2;
DictionarySizeLimits.Ppmd = ppmd;
DictionarySizeLimits.ZStandard = zStandard;
}
}
[Collection(DictionarySizeLimitCollection.Name)]
public class DictionarySizeLimitTests : TestBase
{
[Fact]
public void Static_Limits_Have_Format_Defaults_And_Reject_Nonpositive_Values()
{
using var limits = new DictionarySizeLimitScope();
Assert.Equal(4L << 20, DictionarySizeLimits.Rar4);
Assert.Equal(64L << 20, DictionarySizeLimits.Rar5);
Assert.Equal(64L << 20, DictionarySizeLimits.Lzma);
Assert.Equal(64L << 20, DictionarySizeLimits.Lzma2);
Assert.Equal(64L << 20, DictionarySizeLimits.Ppmd);
Assert.Equal(128L << 20, DictionarySizeLimits.ZStandard);
DictionarySizeLimits.Rar5 = 256L << 20;
Assert.Equal(256L << 20, DictionarySizeLimits.Rar5);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Rar4 = 0);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Rar5 = -1);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Lzma = 0);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Lzma2 = -1);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.Ppmd = 0);
Assert.Throws<ArgumentOutOfRangeException>(() => DictionarySizeLimits.ZStandard = -1);
Assert.Equal(256L << 20, DictionarySizeLimits.Rar5);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma_And_Lzma2_Reject_Excessive_Declared_Dictionaries(bool useAsync)
{
foreach (var properties in new[] { new byte[] { 0x5d, 0, 0, 0, 0x10 }, new byte[] { 30 } })
{
using var input = new MemoryStream();
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await LzmaStream.CreateAsync(properties, input, -1, -1)
);
}
else
{
Assert.Throws<InvalidFormatException>(() => LzmaStream.Create(properties, input));
}
Assert.True(input.CanRead);
Assert.Equal(0, input.Position);
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Direct_Lzma_Decoder_Enforces_Default_Limit(bool useAsync)
{
using var decoder = new Decoder();
decoder.SetDecoderProperties(new byte[] { 0x5d, 0, 0, 0, 0x10 });
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await decoder.CodeAsync(
Stream.Null,
Stream.Null,
0,
1,
null!,
CancellationToken.None
)
);
}
else
{
Assert.Throws<InvalidFormatException>(() =>
decoder.Code(Stream.Null, Stream.Null, 0, 1, null!)
);
}
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task Direct_Lzma_Decoder_Can_Retry_After_Limit_Rejection(bool useAsync, bool train)
{
using var limits = new DictionarySizeLimitScope();
var expected = new byte[128];
using var packed = new MemoryStream();
byte[] properties;
using (
var encoder = LzmaStream.Create(new LzmaEncoderProperties(false, 4096), false, packed)
)
{
properties = (byte[])encoder.Properties.Clone();
encoder.Write(expected, 0, expected.Length);
}
packed.Position = 0;
using var output = new MemoryStream();
using var decoder = new Decoder();
decoder.SetDecoderProperties(properties);
DictionarySizeLimits.Lzma = 4095;
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
{
if (train)
{
await decoder.TrainAsync(Stream.Null);
}
else
{
await decoder.CodeAsync(
packed,
output,
packed.Length,
expected.Length,
null!,
CancellationToken.None
);
}
});
}
else
{
Assert.Throws<InvalidFormatException>(() =>
{
if (train)
{
decoder.Train(Stream.Null);
}
else
{
decoder.Code(packed, output, packed.Length, expected.Length, null!);
}
});
}
Assert.Equal(0, packed.Position);
DictionarySizeLimits.Lzma = 4096;
if (useAsync)
{
if (train)
{
await decoder.TrainAsync(Stream.Null);
}
await decoder.CodeAsync(
packed,
output,
packed.Length,
expected.Length,
null!,
CancellationToken.None
);
}
else
{
if (train)
{
decoder.Train(Stream.Null);
}
decoder.Code(packed, output, packed.Length, expected.Length, null!);
}
Assert.Equal(expected, output.ToArray());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma2_Static_Limit_Is_Inclusive(bool useAsync)
{
using var limits = new DictionarySizeLimitScope();
using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 });
DictionarySizeLimits.Lzma2 = 4095;
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await LzmaStream.CreateAsync(new byte[] { 0 }, input, -1, -1)
);
DictionarySizeLimits.Lzma2 = 4096;
using var decoder = await LzmaStream.CreateAsync(new byte[] { 0 }, input, -1, -1);
var buffer = new byte[1];
Assert.Equal(1, await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None));
Assert.Equal(65, buffer[0]);
}
else
{
Assert.Throws<InvalidFormatException>(() => LzmaStream.Create(new byte[] { 0 }, input));
DictionarySizeLimits.Lzma2 = 4096;
using var decoder = LzmaStream.Create(new byte[] { 0 }, input);
Assert.Equal(65, decoder.ReadByte());
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Lzma2_Encoding_Does_Not_Apply_The_Lzma_Input_Limit(bool useAsync)
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Lzma = 4096;
DictionarySizeLimits.Lzma2 = 8192;
var expected = new byte[150000];
using var packed = new MemoryStream();
byte[] properties;
using (var encoder = new Lzma2EncoderStream(packed, 8192, 32))
{
properties = (byte[])encoder.Properties.Clone();
if (useAsync)
{
await encoder.WriteAsync(expected, 0, expected.Length, CancellationToken.None);
await encoder.DisposeAsync();
}
else
{
encoder.Write(expected, 0, expected.Length);
}
}
Assert.True(packed.Length < expected.Length / 10, $"Encoded {packed.Length} bytes.");
packed.Position = 0;
using var output = new MemoryStream();
if (useAsync)
{
await using var decoder = await LzmaStream.CreateAsync(
properties,
packed,
packed.Length,
expected.Length
);
await decoder.CopyToAsync(output, 81920, CancellationToken.None);
}
else
{
using var decoder = LzmaStream.Create(
properties,
packed,
packed.Length,
expected.Length
);
decoder.CopyTo(output);
}
Assert.Equal(expected, output.ToArray());
}
[Fact]
public void Ppmd_Properties_Reject_Large_Models_Before_Eager_Allocation()
{
Assert.Throws<InvalidFormatException>(() => new PpmdProperties(new byte[] { 0xf5, 0x0f }));
Assert.Throws<InvalidFormatException>(() => new PpmdProperties(256 << 20, 6));
}
[Fact]
public void Ppmd_Properties_Preserve_Size_After_Rejected_Change()
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Ppmd = 2 << 20;
var properties = new PpmdProperties(1 << 20, 6);
var originalHeader = properties.Properties;
Assert.Throws<InvalidFormatException>(() => properties.AllocatorSize = 3 << 20);
Assert.Equal(1 << 20, properties.AllocatorSize);
Assert.Equal(originalHeader, properties.Properties);
properties.AllocatorSize = 2 << 20;
Assert.Equal(2 << 20, properties.AllocatorSize);
Assert.Equal(2 << 20, new PpmdProperties(properties.Properties).AllocatorSize);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task SevenZip_Ppmd_Rejects_Large_Models_Before_Allocation(bool useAsync)
{
var properties = new PpmdProperties(new byte[] { 6, 0, 0, 0, 0x10 });
using var input = new MemoryStream();
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await PpmdStream.CreateAsync(properties, input, false, CancellationToken.None)
);
}
else
{
Assert.Throws<InvalidFormatException>(() =>
PpmdStream.Create(properties, input, false)
);
}
Assert.True(input.CanRead);
}
[Fact]
public void Rar_Ppmd_Suballocator_Enforces_Limit_Before_Renting()
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Ppmd = 1 << 20;
using var allocator = new SubAllocator();
Assert.Throws<InvalidFormatException>(() => allocator.StartSubAllocator(2 << 20));
Assert.Equal(0, allocator.GetAllocatedMemory());
Assert.True(allocator.StartSubAllocator(1 << 20));
Assert.Equal(1 << 20, allocator.GetAllocatedMemory());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task LZip_Uses_Static_Lzma_Limit(bool useAsync)
{
var bytes = new byte[26];
new byte[] { (byte)'L', (byte)'Z', (byte)'I', (byte)'P', 1, 28 }.CopyTo(bytes, 0);
using var input = new MemoryStream(bytes);
if (useAsync)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await LZipStream.CreateAsync(
input,
CompressionMode.Decompress,
cancellationToken: CancellationToken.None
)
);
}
else
{
Assert.Throws<InvalidFormatException>(() =>
LZipStream.Create(input, CompressionMode.Decompress)
);
}
Assert.True(input.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Zstandard_Enforces_Default_And_Exact_Static_Window_Limits(bool useAsync)
{
using var limits = new DictionarySizeLimitScope();
var defaultMaximum = DictionarySizeLimits.ZStandard;
foreach (var maximum in new long?[] { null, 16383, 16384 })
{
DictionarySizeLimits.ZStandard = maximum ?? defaultMaximum;
// A 1 GiB or 16 KiB window with a final empty raw block.
using var input = new MemoryStream(
new byte[]
{
0x28,
0xb5,
0x2f,
0xfd,
0,
maximum is null ? (byte)0xa0 : (byte)0x20,
1,
0,
0,
}
);
using var decoder = new DecompressionStream(input);
var buffer = new byte[1];
if (maximum == 16384)
{
var read = useAsync
? await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None)
: decoder.Read(buffer, 0, 1);
Assert.Equal(0, read);
}
else
{
var exception = useAsync
? await Assert.ThrowsAsync<ZstdException>(async () =>
await decoder.ReadAsync(buffer, 0, 1, CancellationToken.None)
)
: Assert.Throws<ZstdException>(() => decoder.Read(buffer, 0, 1));
Assert.Equal(
ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge,
exception.Code
);
}
}
}
[Fact]
public void Zstandard_Rejects_Unsupported_Limits_Without_Changing_Configuration()
{
using var limits = new DictionarySizeLimitScope();
var original = DictionarySizeLimits.ZStandard;
var maximum = 1L << (IntPtr.Size == 4 ? 30 : 31);
foreach (var invalid in new[] { 512L, 1023, maximum + 1, long.MaxValue })
{
Assert.Throws<ArgumentOutOfRangeException>(() =>
DictionarySizeLimits.ZStandard = invalid
);
Assert.Equal(original, DictionarySizeLimits.ZStandard);
using var decoder = new Decompressor();
}
foreach (var valid in new[] { 1024L, maximum })
{
DictionarySizeLimits.ZStandard = valid;
Assert.Equal(valid, DictionarySizeLimits.ZStandard);
using var decoder = new Decompressor();
}
}
public static IEnumerable<object[]> ArchiveCases()
{
foreach (
var file in new[]
{
"Rar.rar",
"Rar4.rar",
"Rar5.rar",
"Rar5.solid.rar",
"Zip.lzma.zip",
"Zip.ppmd.zip",
"WinZip27_XZ.zipx",
"WinZip27_ZSTD.zipx",
"7Zip.LZMA.7z",
"7Zip.LZMA2.7z",
"7Zip.PPMd.7z",
"7Zip.ZSTD.7z",
"Tar.tar.xz",
"Tar.tar.lz",
"Tar.tar.zst",
}
)
{
foreach (var archiveApi in new[] { false, true })
{
foreach (var useAsync in new[] { false, true })
{
if (
(!archiveApi && file.EndsWith(".7z", StringComparison.Ordinal))
|| (archiveApi && file.StartsWith("Tar.", StringComparison.Ordinal))
)
{
continue;
}
yield return new object[] { file, archiveApi, useAsync };
}
}
}
}
[Theory]
[MemberData(nameof(ArchiveCases))]
public async Task Static_Limits_Apply_To_Archive_And_Reader_Paths(
string file,
bool archiveApi,
bool useAsync
)
{
using (var limits = new DictionarySizeLimitScope())
{
DictionarySizeLimits.Rar4 =
DictionarySizeLimits.Rar5 =
DictionarySizeLimits.Lzma =
DictionarySizeLimits.Lzma2 =
DictionarySizeLimits.Ppmd =
DictionarySizeLimits.ZStandard =
1024;
await Assert.ThrowsAnyAsync<SharpCompressException>(() =>
Extract(file, archiveApi, useAsync)
);
}
await Extract(file, archiveApi, useAsync);
}
private static async Task Extract(string file, bool archiveApi, bool useAsync)
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, file));
var options = ReaderOptions.ForExternalStream;
if (archiveApi && useAsync)
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
input,
options,
CancellationToken.None
);
await foreach (var entry in archive.EntriesAsync)
{
if (!entry.IsDirectory)
{
using var stream = await entry.OpenEntryStreamAsync(CancellationToken.None);
await stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
}
}
else if (archiveApi)
{
using var archive = ArchiveFactory.OpenArchive(input, options);
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
{
using var stream = entry.OpenEntryStream();
stream.CopyTo(Stream.Null);
}
}
else if (useAsync)
{
using var forwardOnly = new NonSeekableStream(input);
await using var reader = await ReaderFactory.OpenAsyncReader(
forwardOnly,
options,
CancellationToken.None
);
while (await reader.MoveToNextEntryAsync(CancellationToken.None))
{
if (!reader.Entry.IsDirectory)
{
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None);
}
}
}
else
{
using var forwardOnly = new NonSeekableStream(input);
using var reader = ReaderFactory.OpenReader(forwardOnly, options);
while (reader.MoveToNextEntry())
{
if (!reader.Entry.IsDirectory)
{
reader.WriteEntryTo(Stream.Null);
}
}
}
}
}

View File

@@ -0,0 +1,74 @@
using System;
using System.IO;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Readers;
using SharpCompress.Writers.SevenZip;
using Xunit;
namespace SharpCompress.Test.Streams;
[Collection(DictionarySizeLimitCollection.Name)]
public class Lzma2ParallelDictionaryLimitTests : TestBase
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public void SevenZip_File_Extraction_Reports_Dictionary_Limits_Consistently(bool parallel)
{
using var limits = new DictionarySizeLimitScope();
var path = Path.Combine(SCRATCH_FILES_PATH, "parallel-dictionary-limit.7z");
using (var output = File.Create(path))
using (
var writer = new SevenZipWriter(
output,
new SevenZipWriterOptions(CompressionType.LZMA2)
{
CompressHeader = false,
LzmaProperties = new LzmaEncoderProperties(false, 8192, 32),
}
)
)
{
// Two full-reset chunks make the file-backed parallel path eligible.
using var source = new MemoryStream(new byte[(2 << 20) + (1 << 16)]);
writer.Write("payload.bin", source, DateTime.UtcNow);
}
DictionarySizeLimits.Lzma2 = 4096;
using var archive = SevenZipArchive.OpenArchive(
path,
ReaderOptions.ForFilePath.WithEnableParallelism(parallel)
);
var entry = Assert.Single(archive.Entries);
Assert.Throws<InvalidFormatException>(() =>
{
using var stream = entry.OpenEntryStream();
stream.CopyTo(Stream.Null);
});
}
#if !LEGACY_DOTNET
[Fact]
public void Parallel_Dictionary_Validation_Runs_Before_Workers()
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Lzma2 = 4096;
var path = Path.Combine(SCRATCH_FILES_PATH, "parallel-dictionary-workers.bin");
using var input = File.Create(path);
using var output = File.Create(path + ".out");
var blocks = new[] { new Lzma2Block(0, 1, 0, 1), new Lzma2Block(1, 1, 1, 1) };
Assert.Throws<InvalidFormatException>(() =>
Lzma2ParallelDecoder.DecodeBlocksParallel(
input.SafeFileHandle,
new byte[] { 2 },
0,
blocks,
output.SafeFileHandle,
2
)
);
Assert.Equal(0, output.Length);
}
#endif
}

View File

@@ -10,6 +10,7 @@ using Xunit;
namespace SharpCompress.Test.Streams;
[Collection(DictionarySizeLimitCollection.Name)]
public class LzmaDictionaryAllocationTests
{
[Theory]
@@ -22,6 +23,8 @@ public class LzmaDictionaryAllocationTests
bool direct
)
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Lzma = int.MaxValue;
var expected = new byte[20000];
new Random(123).NextBytes(expected);
using var packed = new MemoryStream();
@@ -210,6 +213,8 @@ public class LzmaDictionaryAllocationTests
[InlineData(true)]
public async Task Lzma2_Tiny_Stream_With_Huge_Dictionary_Decodes(bool useAsync)
{
using var limits = new DictionarySizeLimitScope();
DictionarySizeLimits.Lzma2 = int.MaxValue;
using var input = new MemoryStream(new byte[] { 1, 0, 0, 65, 0 });
using var output = new MemoryStream();
if (useAsync)

View File

@@ -0,0 +1,93 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Archives.Tar;
using SharpCompress.Common;
using Xunit;
namespace SharpCompress.Test.Tar;
public class TarArchiveDuplicateTests
{
[Fact]
public void Tar_AddEntry_RejectsDuplicateAfterFirstEntry()
{
using var archive = TarArchive.CreateArchive();
using var stream = new MemoryStream();
archive.AddEntry("other", stream, false);
archive.AddEntry("dir/file", stream, false);
Assert.Throws<ArchiveException>(() => archive.AddEntry("DIR\\FILE", stream, false));
var entry = archive.Entries.Single(x => x.Key == "dir/file");
archive.RemoveEntry(entry);
archive.AddEntry("dir/file", stream, false);
}
[Theory]
[InlineData("file.txt", "C:/file.txt", false)]
[InlineData("file.txt", "//file.txt/", false)]
[InlineData("dir/file.txt", "DIR\\FILE.TXT", false)]
[InlineData("file.txt", "C:/file.txt", true)]
[InlineData("file.txt", "//file.txt/", true)]
[InlineData("dir/file.txt", "DIR\\FILE.TXT", true)]
public async Task Tar_AddEntry_Rejects_Writer_Normalized_Duplicates(
string first,
string duplicate,
bool useAsync
)
{
using var stream = new MemoryStream();
if (useAsync)
{
await using var archive = await TarArchive.CreateAsyncArchive();
await archive.AddEntryAsync(first, stream, false);
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddEntryAsync(duplicate, stream, false)
);
}
else
{
using var archive = TarArchive.CreateArchive();
archive.AddEntry(first, stream, false);
Assert.Throws<ArchiveException>(() => archive.AddEntry(duplicate, stream, false));
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Tar_AddDirectory_Rejects_Trailing_Slash_Duplicates(bool useAsync)
{
if (useAsync)
{
await using var archive = await TarArchive.CreateAsyncArchive();
await archive.AddDirectoryEntryAsync("folder");
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddDirectoryEntryAsync("folder/")
);
}
else
{
using var archive = TarArchive.CreateArchive();
archive.AddDirectoryEntry("folder");
Assert.Throws<ArchiveException>(() => archive.AddDirectoryEntry("folder/"));
}
}
[Fact]
public async Task Tar_Async_ChecksPendingEntriesWhileRebuildingIsPaused()
{
await using var archive = await TarArchive.CreateAsyncArchive();
using var stream = new MemoryStream();
using (archive.PauseEntryRebuilding())
{
await archive.AddEntryAsync("other", stream, false);
var entry = await archive.AddEntryAsync("dir/file", stream, false);
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddEntryAsync("DIR\\FILE", stream, false)
);
await archive.RemoveEntryAsync(entry);
await archive.AddEntryAsync("dir/file", stream, false);
}
Assert.Equal(2, (await archive.EntriesAsync.ToListAsync()).Count);
}
}

View File

@@ -0,0 +1,428 @@
using System;
using System.Buffers.Binary;
using System.Globalization;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Common.Tar.Headers;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Test.Mocks;
using Xunit;
namespace SharpCompress.Test.Tar;
public class TarSizeValidationTests
{
[Theory]
[InlineData(-1L, false)]
[InlineData(-512L, false)]
[InlineData(-1024L, false)]
[InlineData(long.MinValue, false)]
[InlineData(long.MaxValue, false)]
[InlineData(TarHeader.MAX_ENTRY_SIZE + 1, false)]
[InlineData(-1L, true)]
[InlineData(-512L, true)]
[InlineData(-1024L, true)]
[InlineData(long.MinValue, true)]
public async Task Tar_Header_Rejects_Invalid_Size(long size, bool negativeMarker)
{
using var stream = new MemoryStream(CreateHeader(size, negativeMarker));
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true);
var header = new TarHeader(new ArchiveEncoding());
Assert.Throws<InvalidFormatException>(() => header.Read(reader));
stream.Position = 0;
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
using var asyncReader = new AsyncBinaryReader(asyncStream, leaveOpen: true);
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await header.ReadAsync(asyncReader, cancellationToken: CancellationToken.None)
);
}
[Theory]
[InlineData(TarHeader.MAX_ENTRY_SIZE - 1)]
[InlineData(TarHeader.MAX_ENTRY_SIZE)]
public async Task Tar_Header_Accepts_Maximum_Paddable_Size(long size)
{
using var stream = new MemoryStream(CreateHeader(size));
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true);
var header = new TarHeader(new ArchiveEncoding());
Assert.True(header.Read(reader));
Assert.Equal(size, header.Size);
stream.Position = 0;
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
using var asyncReader = new AsyncBinaryReader(asyncStream, leaveOpen: true);
Assert.True(await header.ReadAsync(asyncReader, cancellationToken: CancellationToken.None));
Assert.Equal(size, header.Size);
}
[Theory]
[InlineData(124, false)]
[InlineData(125, false)]
[InlineData(126, false)]
[InlineData(127, false)]
[InlineData(124, true)]
[InlineData(125, true)]
[InlineData(126, true)]
[InlineData(127, true)]
public async Task Tar_ArchiveFactory_Rejects_Binary_Size_With_Ignored_High_Bits(
int offset,
bool useAsync
)
{
var header = CreateHeader(0);
// Keep the binary marker, but encode a size larger than a long can hold.
header[offset] |= 0x01;
WriteChecksum(header);
using var stream = new MemoryStream(header);
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await Assert.ThrowsAsync<ArchiveOperationException>(async () =>
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
});
}
else
{
Assert.Throws<ArchiveOperationException>(() =>
{
using var archive = ArchiveFactory.OpenArchive(
stream,
ReaderOptions.ForExternalStream
);
});
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Tar_ArchiveFactory_Rejects_Negative_First_Entry(bool useAsync)
{
// The advisory payload is just one checksum-valid header with size -512.
using var stream = new MemoryStream(CreateHeader(-512));
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await Assert.ThrowsAsync<ArchiveOperationException>(async () =>
{
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
});
}
else
{
Assert.Throws<ArchiveOperationException>(() =>
{
using var archive = ArchiveFactory.OpenArchive(
stream,
ReaderOptions.ForExternalStream
);
});
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(-1L, false)]
[InlineData(-512L, false)]
[InlineData(-1024L, false)]
[InlineData(long.MinValue, false)]
[InlineData(long.MaxValue, false)]
[InlineData(long.MaxValue - 511, false)]
[InlineData(-1L, true)]
[InlineData(-512L, true)]
[InlineData(-1024L, true)]
[InlineData(long.MinValue, true)]
[InlineData(long.MaxValue, true)]
[InlineData(long.MaxValue - 511, true)]
public async Task Tar_Archive_Rejects_Invalid_Entry_Size(long size, bool useAsync)
{
using var stream = CreateArchive(size);
// A valid first entry bypasses format detection so enumeration must validate
// the next header. Only advance twice to keep regressions from looping.
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
await using var entries = archive.EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
Assert.True(await entries.MoveNextAsync());
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
await entries.MoveNextAsync()
);
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream, ReaderOptions.ForExternalStream);
using var entries = archive.Entries.GetEnumerator();
Assert.True(entries.MoveNext());
Assert.Throws<IncompleteArchiveException>(() => entries.MoveNext());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Tar_Reader_Rejects_Negative_Entry_Size(bool useAsync)
{
using var stream = new ForwardOnlyStream(CreateArchive(-512));
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var reader = await ReaderFactory.OpenAsyncReader(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
await reader.MoveToNextEntryAsync(CancellationToken.None)
);
}
else
{
using var reader = ReaderFactory.OpenReader(stream, ReaderOptions.ForExternalStream);
Assert.True(reader.MoveToNextEntry());
Assert.Throws<IncompleteArchiveException>(() => reader.MoveToNextEntry());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(0, false)]
[InlineData(1, false)]
[InlineData(511, false)]
[InlineData(512, false)]
[InlineData(513, false)]
[InlineData(0, true)]
[InlineData(1, true)]
[InlineData(511, true)]
[InlineData(512, true)]
[InlineData(513, true)]
public async Task Tar_Archive_Preserves_Nonnegative_Binary_Sizes(int size, bool useAsync)
{
using var stream = CreateArchive(size);
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var archive = await ArchiveFactory.OpenAsyncArchive(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
await using var entries = archive.EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
Assert.True(await entries.MoveNextAsync());
Assert.Equal(0, entries.Current.Size);
Assert.True(await entries.MoveNextAsync());
Assert.Equal(size, entries.Current.Size);
Assert.False(await entries.MoveNextAsync());
}
else
{
using var archive = ArchiveFactory.OpenArchive(stream, ReaderOptions.ForExternalStream);
using var entries = archive.Entries.GetEnumerator();
Assert.True(entries.MoveNext());
Assert.Equal(0, entries.Current.Size);
Assert.True(entries.MoveNext());
Assert.Equal(size, entries.Current.Size);
Assert.False(entries.MoveNext());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(0, false)]
[InlineData(1, false)]
[InlineData(511, false)]
[InlineData(512, false)]
[InlineData(513, false)]
[InlineData(0, true)]
[InlineData(1, true)]
[InlineData(511, true)]
[InlineData(512, true)]
[InlineData(513, true)]
public async Task Tar_Reader_Preserves_Nonnegative_Binary_Sizes(int size, bool useAsync)
{
using var stream = new ForwardOnlyStream(CreateArchive(size));
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var reader = await ReaderFactory.OpenAsyncReader(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
Assert.Equal(0, reader.Entry.Size);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
Assert.Equal(size, reader.Entry.Size);
Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None));
}
else
{
using var reader = ReaderFactory.OpenReader(stream, ReaderOptions.ForExternalStream);
Assert.True(reader.MoveToNextEntry());
Assert.Equal(0, reader.Entry.Size);
Assert.True(reader.MoveToNextEntry());
Assert.Equal(size, reader.Entry.Size);
Assert.False(reader.MoveToNextEntry());
}
Assert.True(stream.CanRead);
}
[Theory]
[InlineData(long.MaxValue, false)]
[InlineData(long.MaxValue, true)]
[InlineData(TarHeader.MAX_ENTRY_SIZE + 1, false)]
[InlineData(TarHeader.MAX_ENTRY_SIZE + 1, true)]
[InlineData(513L, false)]
[InlineData(513L, true)]
public async Task Tar_Reader_Validates_Effective_Pax_Size(long size, bool useAsync)
{
using var data = CreatePaxArchive(size);
using var stream = new ForwardOnlyStream(data);
var invalid = size > TarHeader.MAX_ENTRY_SIZE;
if (useAsync)
{
using var asyncStream = new AsyncOnlyStream(stream, disposeStream: false);
await using var reader = await ReaderFactory.OpenAsyncReader(
asyncStream,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
if (invalid)
{
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
await reader.MoveToNextEntryAsync(CancellationToken.None)
);
}
else
{
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
Assert.Equal(size, reader.Entry.Size);
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None);
Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None));
}
}
else
{
using var reader = ReaderFactory.OpenReader(stream, ReaderOptions.ForExternalStream);
Assert.True(reader.MoveToNextEntry());
if (invalid)
{
Assert.Throws<IncompleteArchiveException>(() => reader.MoveToNextEntry());
}
else
{
Assert.True(reader.MoveToNextEntry());
Assert.Equal(size, reader.Entry.Size);
reader.WriteEntryTo(Stream.Null);
Assert.False(reader.MoveToNextEntry());
}
}
Assert.True(stream.CanRead);
}
private static MemoryStream CreatePaxArchive(long size)
{
var record = $" size={size.ToString(CultureInfo.InvariantCulture)}\n";
var length = record.Length + 1;
while (length != record.Length + length.ToString(CultureInfo.InvariantCulture).Length)
{
length = record.Length + length.ToString(CultureInfo.InvariantCulture).Length;
}
var payload = Encoding.ASCII.GetBytes(
length.ToString(CultureInfo.InvariantCulture) + record
);
var paxHeader = CreateHeader(payload.Length);
paxHeader[156] = (byte)'x';
WriteChecksum(paxHeader);
var stream = new MemoryStream();
var firstHeader = CreateHeader(0);
stream.Write(firstHeader, 0, firstHeader.Length);
stream.Write(paxHeader, 0, paxHeader.Length);
stream.Write(payload, 0, payload.Length);
stream.SetLength(stream.Length + ((512 - payload.Length % 512) % 512));
stream.Position = stream.Length;
var header = CreateHeader(0);
stream.Write(header, 0, header.Length);
if (size >= 0 && size <= 1024)
{
stream.SetLength(stream.Length + ((size + 511) / 512 * 512));
}
stream.SetLength(stream.Length + 1024);
stream.Position = 0;
return stream;
}
private static MemoryStream CreateArchive(long size)
{
var stream = new MemoryStream();
var firstHeader = CreateHeader(0);
stream.Write(firstHeader, 0, firstHeader.Length);
var header = CreateHeader(size);
stream.Write(header, 0, header.Length);
// Invalid sizes need no payload, and must never cause large test allocations.
if (size >= 0 && size <= 1024)
{
stream.SetLength(stream.Length + ((size + 511) / 512 * 512));
}
stream.SetLength(stream.Length + 1024);
stream.Position = 0;
return stream;
}
private static byte[] CreateHeader(long size, bool negativeMarker = false)
{
var header = new byte[512];
header[0] = (byte)'x';
header[124] = 0x80;
if (negativeMarker)
{
// Cover GNU negative encoding as well as the advisory's 0x80 prefix.
for (var i = 124; i < 128; i++)
{
header[i] = 0xff;
}
}
BinaryPrimitives.WriteInt64BigEndian(header.AsSpan(128, 8), size);
header[156] = (byte)'0';
Encoding.ASCII.GetBytes("ustar\0" + "00").CopyTo(header, 257);
WriteChecksum(header);
return header;
}
private static void WriteChecksum(byte[] header)
{
var checksum = TarHeader.RecalculateChecksum(header);
Encoding
.ASCII.GetBytes(Convert.ToString(checksum, 8).PadLeft(6, '0') + "\0 ")
.CopyTo(header, 148);
}
}

View File

@@ -0,0 +1,354 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Archives.Tar;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
using SharpCompress.Readers;
using SharpCompress.Writers;
using SharpCompress.Writers.Tar;
using SharpCompress.Writers.Zip;
using Xunit;
namespace SharpCompress.Test;
public class WritableArchiveIdentityTests
{
[Theory]
[InlineData(ArchiveType.Zip, false, false)]
[InlineData(ArchiveType.Zip, false, true)]
[InlineData(ArchiveType.Zip, true, false)]
[InlineData(ArchiveType.Zip, true, true)]
[InlineData(ArchiveType.Tar, false, false)]
[InlineData(ArchiveType.Tar, false, true)]
[InlineData(ArchiveType.Tar, true, false)]
[InlineData(ArchiveType.Tar, true, true)]
public async Task Removal_Survives_Switching_Apis_And_Rebuilding(
ArchiveType type,
bool asyncFirst,
bool paused
)
{
using var input = CreateArchive(type, "remove", "keep");
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
var writable = (IWritableArchive)archive;
var asyncWritable = (IWritableAsyncArchive)archive;
// Leave the first loader partially consumed before switching APIs.
var entry = asyncFirst
? await asyncWritable.EntriesAsync.FirstAsync()
: archive.Entries.First();
var otherEntry = asyncFirst
? archive.Entries.First()
: await asyncWritable.EntriesAsync.FirstAsync();
Assert.Same(entry, otherEntry);
using var source = new MemoryStream(new byte[] { 42 });
using (paused ? writable.PauseEntryRebuilding() : null)
{
if (asyncFirst)
{
writable.RemoveEntry(entry);
writable.AddEntry("new", source, false);
}
else
{
await asyncWritable.RemoveEntryAsync(entry);
await asyncWritable.AddEntryAsync(
"new",
source,
false,
cancellationToken: CancellationToken.None
);
}
}
var syncEntries = archive.Entries.ToList();
var asyncEntries = await asyncWritable.EntriesAsync.ToListAsync();
Assert.Equal(new[] { "keep", "new" }, syncEntries.Select(e => e.Key));
Assert.Equal(syncEntries, asyncEntries);
foreach (var useAsync in new[] { false, true })
{
using var output = new MemoryStream();
await Save(archive, output, useAsync);
output.Position = 0;
using var saved = ArchiveFactory.OpenArchive(output, ReaderOptions.ForExternalStream);
Assert.Equal(new[] { "keep", "new" }, saved.Entries.Select(e => e.Key));
}
Assert.True(input.CanRead);
Assert.True(source.CanRead);
}
[Theory]
[InlineData(ArchiveType.Zip, false)]
[InlineData(ArchiveType.Zip, true)]
[InlineData(ArchiveType.Tar, false)]
[InlineData(ArchiveType.Tar, true)]
public async Task Shared_Identity_Does_Not_Remove_Other_Entries_With_The_Same_Name(
ArchiveType type,
bool asyncFirst
)
{
using var input = CreateArchive(type, "same", "same");
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IWritableAsyncArchive)archive;
if (asyncFirst)
{
var entry = await asyncArchive.EntriesAsync.FirstAsync();
((IWritableArchive)archive).RemoveEntry(entry);
}
else
{
await asyncArchive.RemoveEntryAsync(archive.Entries.First());
}
var remaining = Assert.Single(archive.Entries);
Assert.Same(remaining, Assert.Single(await asyncArchive.EntriesAsync.ToListAsync()));
using var payload = remaining.OpenEntryStream();
Assert.Equal(2, payload.ReadByte());
}
[Theory]
[InlineData(ArchiveType.Zip)]
[InlineData(ArchiveType.Tar)]
public async Task Partially_Consumed_Loaders_Resume_From_The_Shared_Cache(ArchiveType type)
{
using var input = CreateArchive(type, "first", "second", "third");
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
using var syncEntries = archive.Entries.GetEnumerator();
await using var asyncEntries = ((IAsyncArchive)archive).EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
Assert.True(syncEntries.MoveNext());
Assert.True(await asyncEntries.MoveNextAsync());
Assert.Same(syncEntries.Current, asyncEntries.Current);
Assert.True(await asyncEntries.MoveNextAsync());
var second = asyncEntries.Current;
Assert.True(syncEntries.MoveNext());
Assert.Same(second, syncEntries.Current);
Assert.True(syncEntries.MoveNext());
Assert.True(await asyncEntries.MoveNextAsync());
Assert.Same(syncEntries.Current, asyncEntries.Current);
Assert.False(syncEntries.MoveNext());
Assert.False(await asyncEntries.MoveNextAsync());
}
[Theory]
[InlineData(ArchiveType.Zip)]
[InlineData(ArchiveType.Tar)]
public async Task Alternating_Loaders_Read_Only_A_Linear_Number_Of_Headers(ArchiveType type)
{
const int count = 128;
var names = Enumerable.Range(0, count).Select(i => $"entry-{i}").ToArray();
using var data = CreateArchive(type, names);
using var input = new CountingReadStream(data);
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
using var syncEntries = archive.Entries.GetEnumerator();
await using var asyncEntries = ((IAsyncArchive)archive).EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
for (var i = 0; i < count; i++)
{
if ((i & 1) == 0)
{
Assert.True(syncEntries.MoveNext());
Assert.True(await asyncEntries.MoveNextAsync());
}
else
{
Assert.True(await asyncEntries.MoveNextAsync());
Assert.True(syncEntries.MoveNext());
}
Assert.Same(syncEntries.Current, asyncEntries.Current);
Assert.Equal(names[i], syncEntries.Current.Key);
}
Assert.False(syncEntries.MoveNext());
Assert.False(await asyncEntries.MoveNextAsync());
// Allow two scans and ZIP's end-of-directory search, but not a prefix
// reparse for every entry. This bound does not depend on timing.
Assert.True(
input.BytesRead <= input.Length * 6,
$"Read {input.BytesRead} bytes for a {input.Length}-byte archive."
);
}
[Theory]
[InlineData(ArchiveType.Zip, false)]
[InlineData(ArchiveType.Zip, true)]
[InlineData(ArchiveType.Tar, false)]
[InlineData(ArchiveType.Tar, true)]
public async Task Partial_Enumerations_Stay_Lazy_And_Reuse_Their_Loader(
ArchiveType type,
bool useAsync
)
{
var names = Enumerable.Range(0, 16).Select(i => $"entry-{i}").ToArray();
using var data = CreateArchive(type, names);
using var input = new CountingReadStream(data);
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
var first = useAsync
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
var readsAfterFirst = input.BytesRead;
var repeated = useAsync
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
Assert.Same(first, repeated);
Assert.Equal(readsAfterFirst, input.BytesRead);
var entries = useAsync
? await asyncArchive.EntriesAsync.ToListAsync()
: archive.Entries.ToList();
Assert.Equal(names, entries.Select(e => e.Key));
Assert.True(
input.BytesRead > readsAfterFirst,
"The first request must not load every header."
);
}
[Theory]
[InlineData(ArchiveType.Zip)]
[InlineData(ArchiveType.Tar)]
public async Task Cancellation_During_A_Read_Does_Not_Lose_An_Entry(ArchiveType type)
{
using var data = CreateArchive(type, "first", "second", "third");
using var input = new CountingReadStream(data);
using var cancellation = new CancellationTokenSource();
await using var archive = await ArchiveFactory.OpenAsyncArchive(
input,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
var first = await archive.EntriesAsync.FirstAsync();
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
{
Assert.True(await entries.MoveNextAsync());
Assert.Same(first, entries.Current);
input.AfterRead = cancellation.Cancel;
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
await entries.MoveNextAsync()
);
}
input.AfterRead = null;
var remaining = await archive.EntriesAsync.ToListAsync();
Assert.Equal(new[] { "first", "second", "third" }, remaining.Select(e => e.Key));
Assert.Same(first, remaining[0]);
}
private sealed class CountingReadStream(Stream source) : Stream
{
public long BytesRead { get; private set; }
public Action? AfterRead { get; set; }
public override bool CanRead => source.CanRead;
public override bool CanSeek => source.CanSeek;
public override bool CanWrite => false;
public override long Length => source.Length;
public override long Position
{
get => source.Position;
set => source.Position = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
var read = source.Read(buffer, offset, count);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
public override async Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
var read = await source.ReadAsync(buffer, offset, count, cancellationToken);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
#if !LEGACY_DOTNET
public override int Read(Span<byte> buffer)
{
var read = source.Read(buffer);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
var read = await source.ReadAsync(buffer, cancellationToken);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
#endif
public override long Seek(long offset, SeekOrigin origin) => source.Seek(offset, origin);
public override void Flush() => source.Flush();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
}
private static MemoryStream CreateArchive(ArchiveType type, params string[] names)
{
var input = new MemoryStream();
using (
var writer = WriterFactory.OpenWriter(
input,
type,
new WriterOptions(CompressionType.None) { LeaveStreamOpen = true }
)
)
{
for (var i = 0; i < names.Length; i++)
{
using var source = new MemoryStream(new[] { (byte)(i + 1) });
writer.Write(names[i], source, DateTime.UtcNow);
}
}
input.Position = 0;
return input;
}
private static async Task Save(IArchive archive, Stream output, bool useAsync)
{
if (archive is ZipArchive zip)
{
var options = new ZipWriterOptions(CompressionType.None) { LeaveStreamOpen = true };
if (useAsync)
{
await zip.SaveToAsync(output, options, CancellationToken.None);
}
else
{
zip.SaveTo(output, options);
}
}
else
{
var tar = (TarArchive)archive;
var options = new TarWriterOptions(CompressionType.None) { LeaveStreamOpen = true };
if (useAsync)
{
await tar.SaveToAsync(output, options, CancellationToken.None);
}
else
{
tar.SaveTo(output, options);
}
}
}
}

View File

@@ -363,11 +363,7 @@ public class Zip64DescriptorSizeTests
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
var first = reader.Entry;
#if LEGACY_DOTNET
using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#else
await using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#endif
{
await stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
@@ -378,11 +374,7 @@ public class Zip64DescriptorSizeTests
}
Assert.Equal("second", reader.Entry.Key);
using var second = new MemoryStream();
#if LEGACY_DOTNET
using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#else
await using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#endif
{
await stream.CopyToAsync(second, 81920, CancellationToken.None);
}

View File

@@ -143,11 +143,7 @@ public class Zip64NonSeekableTests
continue;
}
Assert.Equal(entries[index].Name, reader.Entry.Key);
#if LEGACY_DOTNET
using var source = await reader.OpenEntryStreamAsync(CancellationToken.None);
#else
await using var source = await reader.OpenEntryStreamAsync(CancellationToken.None);
#endif
using var extracted = new MemoryStream();
await source.CopyToAsync(extracted, 81920, CancellationToken.None);
Assert.Equal(entries[index++].Content, extracted.ToArray());

View File

@@ -0,0 +1,122 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
using SharpCompress.Readers;
using SharpCompress.Writers.Zip;
using Xunit;
namespace SharpCompress.Test.Zip;
public class ZipArchiveDuplicateTests
{
[Theory]
[InlineData("dir/file.txt", "DIR/FILE.TXT")]
[InlineData("dir/file.txt", "dir\\file.txt")]
[InlineData("file.txt", "C:/file.txt")]
[InlineData("file.txt", "//file.txt/")]
public void Zip_AddEntry_RejectsDuplicateAfterFirstEntry(string first, string duplicate)
{
using var archive = ZipArchive.CreateArchive();
using var stream = new MemoryStream();
archive.AddEntry("other", stream, false);
archive.AddEntry(first, stream, false);
Assert.Throws<ArchiveException>(() => archive.AddEntry(duplicate, stream, false));
}
[Fact]
public void Zip_PausedRebuilding_ChecksPendingAndRemovedEntries()
{
using var archive = ZipArchive.CreateArchive();
using var stream = new MemoryStream();
using (archive.PauseEntryRebuilding())
{
var entry = archive.AddEntry("dir/file", stream, false);
Assert.Throws<ArchiveException>(() => archive.AddEntry("DIR\\FILE", stream, false));
archive.RemoveEntry(entry);
archive.AddEntry("dir/file", stream, false);
archive.AddDirectoryEntry("folder");
Assert.Throws<ArchiveException>(() => archive.AddDirectoryEntry("FOLDER/"));
}
Assert.Equal(2, archive.Entries.Count());
}
[Fact]
public async Task Zip_Async_ChecksPendingEntriesAndExposesModifications()
{
await using var archive = await ZipArchive.CreateAsyncArchive();
using var stream = new MemoryStream();
await archive.AddEntryAsync("other", stream, false);
var entry = await archive.AddEntryAsync("dir/file", stream, false);
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddEntryAsync("DIR\\FILE", stream, false)
);
Assert.Equal(2, (await archive.EntriesAsync.ToListAsync()).Count);
await archive.RemoveEntryAsync(entry);
Assert.Single(await archive.EntriesAsync.ToListAsync());
await archive.AddEntryAsync("dir/file", stream, false);
using (archive.PauseEntryRebuilding())
{
await archive.AddDirectoryEntryAsync("folder");
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddDirectoryEntryAsync("FOLDER/")
);
}
Assert.Equal(3, (await archive.EntriesAsync.ToListAsync()).Count);
}
[Fact]
public async Task Zip_ExistingEntries_AreCheckedSynchronouslyAndAsynchronously()
{
using var data = new MemoryStream();
using var source = new MemoryStream();
using (var archive = ZipArchive.CreateArchive())
{
archive.AddEntry("other", source, false);
archive.AddEntry("dir/file", source, false);
archive.SaveTo(
data,
new ZipWriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true }
);
}
data.Position = 0;
using (
var archive = ZipArchive.OpenArchive(data, new ReaderOptions { LeaveStreamOpen = true })
)
{
Assert.Throws<ArchiveException>(() => archive.AddEntry("DIR\\FILE", source, false));
}
data.Position = 0;
await using (
var archive = await ZipArchive.OpenAsyncArchive(
data,
new ReaderOptions { LeaveStreamOpen = true }
)
)
{
await Assert.ThrowsAsync<ArchiveException>(async () =>
await archive.AddEntryAsync("DIR\\FILE", source, false)
);
var entry = await archive.EntriesAsync.SingleAsync(x => x.Key == "dir/file");
using (archive.PauseEntryRebuilding())
{
await archive.RemoveEntryAsync(entry);
var temporary = await archive.AddEntryAsync("temporary", source, false);
await archive.RemoveEntryAsync(temporary);
await archive.AddEntryAsync("dir/file", source, false);
Assert.Equal(2, (await archive.EntriesAsync.ToListAsync()).Count);
}
Assert.Equal(2, (await archive.EntriesAsync.ToListAsync()).Count);
using var output = new MemoryStream();
await archive.SaveToAsync(
output,
new ZipWriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true }
);
output.Position = 0;
using var saved = ZipArchive.OpenArchive(output);
Assert.Equal(2, saved.Entries.Count());
Assert.DoesNotContain(saved.Entries, x => x.Key == "temporary");
}
}
}

View File

@@ -0,0 +1,163 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Readers;
using SharpCompress.Test.Mocks;
using Xunit;
namespace SharpCompress.Test.Zip;
public class ZipBZip2CorruptionTests : TestBase
{
[Theory]
[InlineData(240, 0x8d, 0x5a, false)]
[InlineData(11318, 0x35, 0xa8, false)]
[InlineData(41884, 0x12, 0x89, false)]
[InlineData(240, 0x8d, 0x5a, true)]
[InlineData(11318, 0x35, 0xa8, true)]
[InlineData(41884, 0x12, 0x89, true)]
public void Zip_BZip2_Reader_Throws_On_Corruption(
int offset,
byte original,
byte replacement,
bool skipEntry
)
{
using var stream = new ForwardOnlyStream(
CreateCorruptArchive(offset, original, replacement)
);
using var reader = ReaderFactory.OpenReader(stream, ReaderOptions.ForExternalStream);
var exception = Assert.ThrowsAny<SharpCompressException>(() =>
{
while (reader.MoveToNextEntry())
{
// Disposal must also terminate when a failed read leaves the decoder between states.
using var entry = reader.OpenEntryStream();
if (!skipEntry)
{
entry.CopyTo(Stream.Null);
}
}
});
Assert.StartsWith("BZip2", exception.Message);
if (!skipEntry)
{
Assert.IsType<InvalidFormatException>(exception);
}
}
[Theory]
[InlineData(240, 0x8d, 0x5a, false)]
[InlineData(11318, 0x35, 0xa8, false)]
[InlineData(41884, 0x12, 0x89, false)]
[InlineData(240, 0x8d, 0x5a, true)]
[InlineData(11318, 0x35, 0xa8, true)]
[InlineData(41884, 0x12, 0x89, true)]
public async Task Zip_BZip2_Reader_Throws_On_Corruption_Async(
int offset,
byte original,
byte replacement,
bool skipEntry
)
{
using var stream = new AsyncOnlyStream(
new ForwardOnlyStream(CreateCorruptArchive(offset, original, replacement))
);
await using var reader = await ReaderFactory.OpenAsyncReader(
stream,
ReaderOptions.ForExternalStream,
cancellationToken: CancellationToken.None
);
var exception = await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
{
while (await reader.MoveToNextEntryAsync(CancellationToken.None))
{
await using var entry = await reader.OpenEntryStreamAsync(CancellationToken.None);
if (!skipEntry)
{
await entry.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
}
});
Assert.StartsWith("BZip2", exception.Message);
if (!skipEntry)
{
Assert.IsType<InvalidFormatException>(exception);
}
}
[Theory]
[InlineData(240, 0x8d, 0x5a)]
[InlineData(11318, 0x35, 0xa8)]
[InlineData(41884, 0x12, 0x89)]
public void Zip_BZip2_Archive_Throws_On_Corruption(int offset, byte original, byte replacement)
{
using var stream = CreateCorruptArchive(offset, original, replacement);
using var archive = ArchiveFactory.OpenArchive(stream, ReaderOptions.ForExternalStream);
Assert.ThrowsAny<SharpCompressException>(() =>
{
foreach (var entry in archive.Entries)
{
if (!entry.IsDirectory)
{
using var entryStream = entry.OpenEntryStream();
entryStream.CopyTo(Stream.Null);
}
}
});
}
[Theory]
[InlineData(240, 0x8d, 0x5a)]
[InlineData(11318, 0x35, 0xa8)]
[InlineData(41884, 0x12, 0x89)]
public async Task Zip_BZip2_Archive_Throws_On_Corruption_Async(
int offset,
byte original,
byte replacement
)
{
using var stream = new AsyncOnlyStream(CreateCorruptArchive(offset, original, replacement));
await using var archive = await ArchiveFactory.OpenAsyncArchive(
stream,
ReaderOptions.ForExternalStream,
cancellationToken: CancellationToken.None
);
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
{
await foreach (var entry in archive.EntriesAsync)
{
if (!entry.IsDirectory)
{
#if LEGACY_DOTNET
using var entryStream = await entry.OpenEntryStreamAsync(
CancellationToken.None
);
#else
await using var entryStream = await entry.OpenEntryStreamAsync(
CancellationToken.None
);
#endif
await entryStream.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
}
});
}
private static MemoryStream CreateCorruptArchive(int offset, byte original, byte replacement)
{
// Mutate the existing fixture in memory so each regression differs by exactly one byte.
var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Zip.bzip2.dd.zip"));
Assert.Equal(original, bytes[offset]);
bytes[offset] = replacement;
return new MemoryStream(bytes);
}
}

Binary file not shown.