starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-11-15 17:02:03 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Merge pull request #1473 from adamhathcock/review/fix-zip64-descriptor-width

Browse Source 
Fix ZIP64 streaming data descriptor width parsing
This commit is contained in:
Adam Hathcock
2026-10-26 11:36:16 +00:00
committed by GitHub
parent 8fa17607d8 9c555c41cc
commit 2fa1dec513
5 changed files with 408 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/FORMATS.md
View File

@@ -39,6 +39,7 @@
- ZIP/ZIPX companion volumes are discovered with a case-insensitive filename fallback, including on case-sensitive filesystems. An exact generated filename takes precedence; ambiguous fallback matches throw `InvalidFormatException`.
- WinZip AES authentication (HMAC-SHA1-80 over ciphertext) is verified independently of `CheckCrc`. Incorrect tags throw `InvalidFormatException`; truncated ciphertext or tags throw `IncompleteArchiveException`. Reading an entry to completion or disposing its stream verifies authentication.
- ZipReader processes entries from LocalEntry headers (which include directory entries ending with `/`) and intentionally skips DirectoryEntry headers from the central directory, as they are redundant in streaming mode - all entry data comes from LocalEntry headers which ZipReader has already processed.
- Streaming reads honor local-header ZIP64 metadata when reading data descriptors, using 8-byte sizes even for small entries or sizes whose low word equals a record signature. When that metadata is absent, legacy 32-/64-bit descriptor detection is retained. Descriptor size words are values, not ZIP64 sentinels.
- ZIP supports ZStandard for reading and writing. Tar.Zstandard support is read/decompress only.
### Rar Format Notes

15
src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs
View File

@@ -56,6 +56,15 @@ internal partial class LocalEntryHeader : ZipFileEntry
}
var zip64ExtraData = Extra.OfType<Zip64ExtendedInformationExtraField>().FirstOrDefault();
// Remember ZIP64 before the extra field replaces size sentinels with values
// that may be small (or zero placeholders). Descriptor width must not depend
// on those values, nor on Version, which other compression methods also raise.
UsesZip64Sizes =
CompressedSize == uint.MaxValue
|| UncompressedSize == uint.MaxValue
|| zip64ExtraData is not null;
if (zip64ExtraData != null)
{
zip64ExtraData.Process(UncompressedSize, CompressedSize, 0, 0);
@@ -102,4 +111,10 @@ internal partial class LocalEntryHeader : ZipFileEntry
}
internal ushort Version { get; private set; }
/// <summary>
/// Whether the local header declares ZIP64 sizes, requiring 8-byte descriptor sizes
/// even for small entries (APPNOTE 4.3.9.2).
/// </summary>
internal bool UsesZip64Sizes { get; private set; }
}

73
src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs
View File

@@ -128,44 +128,57 @@ internal sealed partial class StreamingZipHeaderFactory
lastEntryHeader.Crc = crc;
lastEntryHeader.IsCrcAvailable = true;
//attempt 32bit read
ulong compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
ulong uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
ulong compressedSize;
ulong uncompressedSize;
//check for zip64 sentinel or unexpected header
bool isSentinel =
compressedSize == 0xFFFFFFFF || uncompressedSize == 0xFFFFFFFF;
bool isHeader = headerBytes == 0x04034b50 || headerBytes == 0x02014b50;
if (!isHeader && !isSentinel)
// A ZIP64 size's low word can equal a header signature. Honor the
// declared width rather than mistaking that word for the next header,
// even if the record following the descriptor is unrecognized.
if (lastEntryHeader.UsesZip64Sizes)
{
//reshuffle into 64-bit values
compressedSize = (uncompressedSize << 32) | compressedSize;
// headerBytes holds the low word; the high word follows on the wire.
var highUncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
uncompressedSize = headerBytes | ((ulong)highUncompressedSize << 32);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else if (isSentinel)
{
//standards-compliant zip64 descriptor
compressedSize = await _reader
.ReadUInt64Async(_cancellationToken)
.ConfigureAwait(false);
uncompressedSize = await _reader
.ReadUInt64Async(_cancellationToken)
.ConfigureAwait(false);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else
{
// Without ZIP64 metadata, retain compatibility with older writers
// that emitted 64-bit descriptors without declaring them locally.
compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
// Keep the legacy probe limited to local/central headers: other
// record signatures can be the low word of a 64-bit size.
if (
headerBytes != ENTRY_HEADER_BYTES
&& headerBytes != DIRECTORY_START_HEADER_BYTES
)
{
// Descriptor words are size data, including 0xFFFFFFFF; unlike
// local headers, descriptors do not contain size sentinels.
compressedSize = (uncompressedSize << 32) | compressedSize;
// headerBytes holds the low word; the high word follows on the wire.
var highUncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
uncompressedSize = headerBytes | ((ulong)highUncompressedSize << 32);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
}
lastEntryHeader.CompressedSize = (long)compressedSize;

48
src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs
View File

@@ -68,28 +68,40 @@ internal sealed partial class StreamingZipHeaderFactory : ZipHeaderFactory
_lastEntryHeader.Crc = crc;
_lastEntryHeader.IsCrcAvailable = true;
//attempt 32bit read
ulong compSize = reader.ReadUInt32();
ulong uncompSize = reader.ReadUInt32();
headerBytes = reader.ReadUInt32();
ulong compSize;
ulong uncompSize;
//check for zip64 sentinel or unexpected header
bool isSentinel = compSize == 0xFFFFFFFF || uncompSize == 0xFFFFFFFF;
bool isHeader = headerBytes == 0x04034b50 || headerBytes == 0x02014b50;
if (!isHeader && !isSentinel)
// A ZIP64 size's low word can equal a header signature. Honor the
// declared width rather than mistaking that word for the next header,
// even if the record following the descriptor is unrecognized.
if (_lastEntryHeader.UsesZip64Sizes)
{
//reshuffle into 64-bit values
compSize = (uncompSize << 32) | compSize;
// headerBytes holds the low word; the high word follows on the wire.
uncompSize = headerBytes | ((ulong)reader.ReadUInt32() << 32);
headerBytes = reader.ReadUInt32();
}
else if (isSentinel)
{
//standards-compliant zip64 descriptor
compSize = reader.ReadUInt64();
uncompSize = reader.ReadUInt64();
headerBytes = reader.ReadUInt32();
}
else
{
// Without ZIP64 metadata, retain compatibility with older writers
// that emitted 64-bit descriptors without declaring them locally.
compSize = reader.ReadUInt32();
uncompSize = reader.ReadUInt32();
headerBytes = reader.ReadUInt32();
// Keep the legacy probe limited to local/central headers: other
// record signatures can be the low word of a 64-bit size.
if (
headerBytes != ENTRY_HEADER_BYTES
&& headerBytes != DIRECTORY_START_HEADER_BYTES
)
{
// Descriptor words are size data, including 0xFFFFFFFF; unlike
// local headers, descriptors do not contain size sentinels.
compSize = (uncompSize << 32) | compSize;
// headerBytes holds the low word; the high word follows on the wire.
uncompSize = headerBytes | ((ulong)reader.ReadUInt32() << 32);
headerBytes = reader.ReadUInt32();
}
}
_lastEntryHeader.CompressedSize = (long)compSize;

329
tests/SharpCompress.Test/Zip/Zip64DescriptorSizeTests.cs
View File

@@ -15,6 +15,8 @@ namespace SharpCompress.Test.Zip;
public class Zip64DescriptorSizeTests
{
private const uint DataDescriptorSignature = 0x08074b50;
public static IEnumerable<object[]> Sizes()
{
foreach (var useAsync in new[] { false, true })
@@ -31,6 +33,13 @@ public class Zip64DescriptorSizeTests
0x100000000UL,
0x10000007BUL,
0x123456789UL,
// The low 32 bits of a ZIP64 descriptor's uncompressed size are read
// as a candidate record signature. These values collide with the local
// file header and the central directory header signatures.
0x04034B50UL,
0x02014B50UL,
0x104034B50UL,
0x102014B50UL,
}
)
{
@@ -48,6 +57,89 @@ public class Zip64DescriptorSizeTests
ulong size
)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
// Parser-only metadata probe: advertise a large size without allocating a large payload.
// The Archive API is intentionally not used: this synthetic descriptor differs from the directory.
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size);
if (!signature)
{
zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray();
}
var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync);
Assert.Equal((long)size, firstSize);
Assert.Equal("second", secondKey);
}
public static IEnumerable<object[]> LegacyDescriptorSizes()
{
foreach (var useAsync in new[] { false, true })
{
foreach (var signature in new[] { false, true })
{
foreach (
var size in new[]
{
0x08074B50UL,
0x05054B50UL,
0x06054B50UL,
0x30304B50UL,
0x06064B50UL,
0x07064B50UL,
}
)
{
yield return new object[] { useAsync, signature, size };
yield return new object[] { useAsync, signature, size | (1UL << 32) };
}
}
}
}
[Theory]
[MemberData(nameof(LegacyDescriptorSizes))]
public async Task Legacy_Descriptor_Preserves_Signature_Size_And_Next_Header(
bool useAsync,
bool signature,
ulong size
)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size);
// Simulate older writers that used 64-bit descriptors without declaring ZIP64.
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0);
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0);
var extra = 30 + BinaryPrimitives.ReadUInt16LittleEndian(zip.AsSpan(26));
BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(extra), 0xFFFF);
if (!signature)
{
zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray();
}
var (firstSize, secondKey) = await ReadFirstTwoEntries(
zip,
useAsync,
expectedCompressedSize: 3
);
Assert.Equal((long)size, firstSize);
Assert.Equal("second", secondKey);
}
[Theory]
[InlineData(false, false, 0x04034B50)]
[InlineData(false, true, 0x04034B50)]
[InlineData(true, false, 0x04034B50)]
[InlineData(true, true, 0x04034B50)]
[InlineData(false, false, 0x02014B50)]
[InlineData(false, true, 0x02014B50)]
[InlineData(true, false, 0x02014B50)]
[InlineData(true, true, 0x02014B50)]
public async Task Signature_Size_RoundTrips(bool useAsync, bool signature, int size)
{
// Write real payloads in small chunks: unlike the metadata probes, the local
// headers, descriptors and central directory all describe the same entry.
using var output = new MemoryStream();
using (
var writer = new ZipWriter(
@@ -56,22 +148,150 @@ public class Zip64DescriptorSizeTests
)
)
{
using var first = new MemoryStream([42]);
writer.Write("first", first, DateTime.Now);
using (var first = writer.WriteToStream("first", new ZipWriterEntryOptions()))
{
var buffer = new byte[8192];
for (var remaining = size; remaining > 0; )
{
var count = Math.Min(remaining, buffer.Length);
first.Write(buffer, 0, count);
remaining -= count;
}
}
using var second = new MemoryStream([43]);
writer.Write("second", second, DateTime.Now);
}
var zip = output.ToArray();
var descriptor = Enumerable
.Range(0, zip.Length - 24)
.First(i => BinaryPrimitives.ReadUInt32LittleEndian(zip.AsSpan(i)) == 0x08074b50);
// Parser-only metadata probe: advertise a large size without allocating a large payload.
// The Archive API is intentionally not used: this synthetic descriptor differs from the directory.
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size);
if (!signature)
{
var descriptor = FindDescriptor(zip);
zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray();
}
var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync);
Assert.Equal(size, firstSize);
Assert.Equal("second", secondKey);
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task Classic_Descriptor_With_High_Version_Is_Read(bool useAsync, bool signature)
{
var zip = WriteZip64Archive(useZip64: false);
var descriptor = FindDescriptor(zip);
// Higher versions are also used by non-ZIP64 compression methods. Version alone
// must not select 64-bit descriptor sizes.
BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(4), 63);
if (!signature)
{
zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray();
}
var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync);
Assert.Equal(1L, firstSize);
Assert.Equal("second", secondKey);
}
public static IEnumerable<object[]> CompressedSizes()
{
foreach (var useAsync in new[] { false, true })
{
foreach (var signature in new[] { false, true })
{
foreach (var zip64Header in new[] { false, true })
{
foreach (var size in new[] { 0xFFFFFFFFUL, 0x1FFFFFFFFUL })
{
yield return new object[] { useAsync, signature, zip64Header, size };
}
}
}
}
}
[Theory]
[MemberData(nameof(CompressedSizes))]
public async Task Descriptor_Preserves_Compressed_Size_Words(
bool useAsync,
bool signature,
bool zip64Header,
ulong size
)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
// Only the descriptor is changed: decompression still consumes the tiny payload.
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 8), size);
if (!zip64Header)
{
// Preserve support for older writers that emitted 64-bit descriptors without
// declaring ZIP64 in the local header.
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0);
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0);
var extra = 30 + BinaryPrimitives.ReadUInt16LittleEndian(zip.AsSpan(26));
BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(extra), 0xFFFF);
}
if (!signature)
{
zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray();
}
await ReadFirstTwoEntries(zip, useAsync, expectedCompressedSize: (long)size);
}
/// <summary>
/// The descriptor width is also resolved when the archive is read from a seekable
/// stream, where the entry data position is calculated from the descriptor sizes.
/// </summary>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Descriptor_Size_Collision_Is_Resolved_On_Seekable_Stream(bool useAsync)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), 0x04034B50UL);
var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync, seekable: true);
Assert.Equal(0x04034B50L, firstSize);
Assert.Equal("second", secondKey);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Zip64_Extra_Field_Declares_Width_With_Zero_Local_Sizes(bool useAsync)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0);
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0);
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), 0x02014B50UL);
var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync);
Assert.Equal(0x02014B50L, firstSize);
Assert.Equal("second", secondKey);
}
/// <summary>
/// A record with an unknown signature after a ZIP64 descriptor ends the enumeration,
/// but the sizes of the entry that was read are kept.
/// </summary>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Unknown_Record_After_Zip64_Descriptor_Keeps_Sizes(bool useAsync)
{
var zip = WriteZip64Archive();
var descriptor = FindDescriptor(zip);
const long size = 0x04034B50;
BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), (ulong)size);
// Corrupt the signature of the local header that follows the first descriptor.
BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(descriptor + 24), 0x04034B51);
using var input = new ForwardOnlyStream(new MemoryStream(zip));
if (useAsync)
{
@@ -82,6 +302,67 @@ public class Zip64DescriptorSizeTests
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
var first = reader.Entry;
Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None));
Assert.Equal(size, first.Size);
}
else
{
using var reader = ReaderFactory.OpenReader(input, ReaderOptions.ForExternalStream);
Assert.True(reader.MoveToNextEntry());
var first = reader.Entry;
Assert.False(reader.MoveToNextEntry());
Assert.Equal(size, first.Size);
}
}
/// <summary>
/// Writes two entries to a forward-only stream, which makes the writer use data
/// descriptors and reserve the ZIP64 fields in the local headers.
/// </summary>
private static byte[] WriteZip64Archive(bool useZip64 = true)
{
using var output = new MemoryStream();
using (
var writer = new ZipWriter(
new ForwardOnlyStream(output),
new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = useZip64 }
)
)
{
using var first = new MemoryStream([42]);
writer.Write("first", first, DateTime.Now);
using var second = new MemoryStream([43]);
writer.Write("second", second, DateTime.Now);
}
return output.ToArray();
}
private static int FindDescriptor(byte[] zip) =>
Enumerable
.Range(0, zip.Length - 24)
.First(i =>
BinaryPrimitives.ReadUInt32LittleEndian(zip.AsSpan(i)) == DataDescriptorSignature
);
private static async Task<(long FirstSize, string? SecondKey)> ReadFirstTwoEntries(
byte[] zip,
bool useAsync,
bool seekable = false,
long? expectedCompressedSize = null
)
{
using var input = seekable
? (Stream)new MemoryStream(zip)
: new ForwardOnlyStream(new MemoryStream(zip));
if (useAsync)
{
await using var reader = await ReaderFactory.OpenAsyncReader(
seekable ? input : new AsyncOnlyStream(input),
ReaderOptions.ForExternalStream,
CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
var first = reader.Entry;
#if LEGACY_DOTNET
using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#else
@@ -91,8 +372,24 @@ public class Zip64DescriptorSizeTests
await stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None);
}
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
Assert.Equal((long)size, first.Size);
if (expectedCompressedSize.HasValue)
{
Assert.Equal(expectedCompressedSize.Value, first.CompressedSize);
}
Assert.Equal("second", reader.Entry.Key);
using var second = new MemoryStream();
#if LEGACY_DOTNET
using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#else
await using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None))
#endif
{
await stream.CopyToAsync(second, 81920, CancellationToken.None);
}
Assert.Equal(new byte[] { 43 }, second.ToArray());
var secondKey = reader.Entry.Key;
Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None));
return (first.Size, secondKey);
}
else
{
@@ -104,8 +401,20 @@ public class Zip64DescriptorSizeTests
stream.CopyTo(Stream.Null);
}
Assert.True(reader.MoveToNextEntry());
Assert.Equal((long)size, first.Size);
if (expectedCompressedSize.HasValue)
{
Assert.Equal(expectedCompressedSize.Value, first.CompressedSize);
}
Assert.Equal("second", reader.Entry.Key);
using var second = new MemoryStream();
using (var stream = reader.OpenEntryStream())
{
stream.CopyTo(second);
}
Assert.Equal(new byte[] { 43 }, second.ToArray());
var secondKey = reader.Entry.Key;
Assert.False(reader.MoveToNextEntry());
return (first.Size, secondKey);
}
}
}