diff --git a/docs/FORMATS.md b/docs/FORMATS.md index 6ad30617..c4c53444 100644 --- a/docs/FORMATS.md +++ b/docs/FORMATS.md @@ -39,6 +39,7 @@ - ZIP/ZIPX companion volumes are discovered with a case-insensitive filename fallback, including on case-sensitive filesystems. An exact generated filename takes precedence; ambiguous fallback matches throw `InvalidFormatException`. - WinZip AES authentication (HMAC-SHA1-80 over ciphertext) is verified independently of `CheckCrc`. Incorrect tags throw `InvalidFormatException`; truncated ciphertext or tags throw `IncompleteArchiveException`. Reading an entry to completion or disposing its stream verifies authentication. - ZipReader processes entries from LocalEntry headers (which include directory entries ending with `/`) and intentionally skips DirectoryEntry headers from the central directory, as they are redundant in streaming mode - all entry data comes from LocalEntry headers which ZipReader has already processed. +- Streaming reads honor local-header ZIP64 metadata when reading data descriptors, using 8-byte sizes even for small entries or sizes whose low word equals a record signature. When that metadata is absent, legacy 32-/64-bit descriptor detection is retained. Descriptor size words are values, not ZIP64 sentinels. - ZIP supports ZStandard for reading and writing. Tar.Zstandard support is read/decompress only. ### Rar Format Notes diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs index 9d4512b1..71411047 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs @@ -56,6 +56,15 @@ internal partial class LocalEntryHeader : ZipFileEntry } var zip64ExtraData = Extra.OfType().FirstOrDefault(); + + // Remember ZIP64 before the extra field replaces size sentinels with values + // that may be small (or zero placeholders). Descriptor width must not depend + // on those values, nor on Version, which other compression methods also raise. + UsesZip64Sizes = + CompressedSize == uint.MaxValue + || UncompressedSize == uint.MaxValue + || zip64ExtraData is not null; + if (zip64ExtraData != null) { zip64ExtraData.Process(UncompressedSize, CompressedSize, 0, 0); @@ -102,4 +111,10 @@ internal partial class LocalEntryHeader : ZipFileEntry } internal ushort Version { get; private set; } + + /// + /// Whether the local header declares ZIP64 sizes, requiring 8-byte descriptor sizes + /// even for small entries (APPNOTE 4.3.9.2). + /// + internal bool UsesZip64Sizes { get; private set; } } diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs index 47d336c9..fdc534e4 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs @@ -128,44 +128,57 @@ internal sealed partial class StreamingZipHeaderFactory lastEntryHeader.Crc = crc; lastEntryHeader.IsCrcAvailable = true; - //attempt 32bit read - ulong compressedSize = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - ulong uncompressedSize = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - headerBytes = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); + ulong compressedSize; + ulong uncompressedSize; - //check for zip64 sentinel or unexpected header - bool isSentinel = - compressedSize == 0xFFFFFFFF || uncompressedSize == 0xFFFFFFFF; - bool isHeader = headerBytes == 0x04034b50 || headerBytes == 0x02014b50; - - if (!isHeader && !isSentinel) + // A ZIP64 size's low word can equal a header signature. Honor the + // declared width rather than mistaking that word for the next header, + // even if the record following the descriptor is unrecognized. + if (lastEntryHeader.UsesZip64Sizes) { - //reshuffle into 64-bit values - compressedSize = (uncompressedSize << 32) | compressedSize; - // headerBytes holds the low word; the high word follows on the wire. - var highUncompressedSize = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - uncompressedSize = headerBytes | ((ulong)highUncompressedSize << 32); - headerBytes = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - } - else if (isSentinel) - { - //standards-compliant zip64 descriptor compressedSize = await _reader .ReadUInt64Async(_cancellationToken) .ConfigureAwait(false); uncompressedSize = await _reader .ReadUInt64Async(_cancellationToken) .ConfigureAwait(false); + headerBytes = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + } + else + { + // Without ZIP64 metadata, retain compatibility with older writers + // that emitted 64-bit descriptors without declaring them locally. + compressedSize = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + uncompressedSize = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + headerBytes = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + + // Keep the legacy probe limited to local/central headers: other + // record signatures can be the low word of a 64-bit size. + if ( + headerBytes != ENTRY_HEADER_BYTES + && headerBytes != DIRECTORY_START_HEADER_BYTES + ) + { + // Descriptor words are size data, including 0xFFFFFFFF; unlike + // local headers, descriptors do not contain size sentinels. + compressedSize = (uncompressedSize << 32) | compressedSize; + // headerBytes holds the low word; the high word follows on the wire. + var highUncompressedSize = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + uncompressedSize = headerBytes | ((ulong)highUncompressedSize << 32); + headerBytes = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + } } lastEntryHeader.CompressedSize = (long)compressedSize; diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs index a22cd55d..15c4b87e 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs @@ -68,28 +68,40 @@ internal sealed partial class StreamingZipHeaderFactory : ZipHeaderFactory _lastEntryHeader.Crc = crc; _lastEntryHeader.IsCrcAvailable = true; - //attempt 32bit read - ulong compSize = reader.ReadUInt32(); - ulong uncompSize = reader.ReadUInt32(); - headerBytes = reader.ReadUInt32(); + ulong compSize; + ulong uncompSize; - //check for zip64 sentinel or unexpected header - bool isSentinel = compSize == 0xFFFFFFFF || uncompSize == 0xFFFFFFFF; - bool isHeader = headerBytes == 0x04034b50 || headerBytes == 0x02014b50; - - if (!isHeader && !isSentinel) + // A ZIP64 size's low word can equal a header signature. Honor the + // declared width rather than mistaking that word for the next header, + // even if the record following the descriptor is unrecognized. + if (_lastEntryHeader.UsesZip64Sizes) { - //reshuffle into 64-bit values - compSize = (uncompSize << 32) | compSize; - // headerBytes holds the low word; the high word follows on the wire. - uncompSize = headerBytes | ((ulong)reader.ReadUInt32() << 32); - headerBytes = reader.ReadUInt32(); - } - else if (isSentinel) - { - //standards-compliant zip64 descriptor compSize = reader.ReadUInt64(); uncompSize = reader.ReadUInt64(); + headerBytes = reader.ReadUInt32(); + } + else + { + // Without ZIP64 metadata, retain compatibility with older writers + // that emitted 64-bit descriptors without declaring them locally. + compSize = reader.ReadUInt32(); + uncompSize = reader.ReadUInt32(); + headerBytes = reader.ReadUInt32(); + + // Keep the legacy probe limited to local/central headers: other + // record signatures can be the low word of a 64-bit size. + if ( + headerBytes != ENTRY_HEADER_BYTES + && headerBytes != DIRECTORY_START_HEADER_BYTES + ) + { + // Descriptor words are size data, including 0xFFFFFFFF; unlike + // local headers, descriptors do not contain size sentinels. + compSize = (uncompSize << 32) | compSize; + // headerBytes holds the low word; the high word follows on the wire. + uncompSize = headerBytes | ((ulong)reader.ReadUInt32() << 32); + headerBytes = reader.ReadUInt32(); + } } _lastEntryHeader.CompressedSize = (long)compSize; diff --git a/tests/SharpCompress.Test/Zip/Zip64DescriptorSizeTests.cs b/tests/SharpCompress.Test/Zip/Zip64DescriptorSizeTests.cs index 872fefb7..f4aeb6ce 100644 --- a/tests/SharpCompress.Test/Zip/Zip64DescriptorSizeTests.cs +++ b/tests/SharpCompress.Test/Zip/Zip64DescriptorSizeTests.cs @@ -15,6 +15,8 @@ namespace SharpCompress.Test.Zip; public class Zip64DescriptorSizeTests { + private const uint DataDescriptorSignature = 0x08074b50; + public static IEnumerable Sizes() { foreach (var useAsync in new[] { false, true }) @@ -31,6 +33,13 @@ public class Zip64DescriptorSizeTests 0x100000000UL, 0x10000007BUL, 0x123456789UL, + // The low 32 bits of a ZIP64 descriptor's uncompressed size are read + // as a candidate record signature. These values collide with the local + // file header and the central directory header signatures. + 0x04034B50UL, + 0x02014B50UL, + 0x104034B50UL, + 0x102014B50UL, } ) { @@ -48,6 +57,89 @@ public class Zip64DescriptorSizeTests ulong size ) { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + // Parser-only metadata probe: advertise a large size without allocating a large payload. + // The Archive API is intentionally not used: this synthetic descriptor differs from the directory. + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size); + if (!signature) + { + zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray(); + } + + var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync); + Assert.Equal((long)size, firstSize); + Assert.Equal("second", secondKey); + } + + public static IEnumerable LegacyDescriptorSizes() + { + foreach (var useAsync in new[] { false, true }) + { + foreach (var signature in new[] { false, true }) + { + foreach ( + var size in new[] + { + 0x08074B50UL, + 0x05054B50UL, + 0x06054B50UL, + 0x30304B50UL, + 0x06064B50UL, + 0x07064B50UL, + } + ) + { + yield return new object[] { useAsync, signature, size }; + yield return new object[] { useAsync, signature, size | (1UL << 32) }; + } + } + } + } + + [Theory] + [MemberData(nameof(LegacyDescriptorSizes))] + public async Task Legacy_Descriptor_Preserves_Signature_Size_And_Next_Header( + bool useAsync, + bool signature, + ulong size + ) + { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size); + // Simulate older writers that used 64-bit descriptors without declaring ZIP64. + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0); + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0); + var extra = 30 + BinaryPrimitives.ReadUInt16LittleEndian(zip.AsSpan(26)); + BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(extra), 0xFFFF); + if (!signature) + { + zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray(); + } + + var (firstSize, secondKey) = await ReadFirstTwoEntries( + zip, + useAsync, + expectedCompressedSize: 3 + ); + Assert.Equal((long)size, firstSize); + Assert.Equal("second", secondKey); + } + + [Theory] + [InlineData(false, false, 0x04034B50)] + [InlineData(false, true, 0x04034B50)] + [InlineData(true, false, 0x04034B50)] + [InlineData(true, true, 0x04034B50)] + [InlineData(false, false, 0x02014B50)] + [InlineData(false, true, 0x02014B50)] + [InlineData(true, false, 0x02014B50)] + [InlineData(true, true, 0x02014B50)] + public async Task Signature_Size_RoundTrips(bool useAsync, bool signature, int size) + { + // Write real payloads in small chunks: unlike the metadata probes, the local + // headers, descriptors and central directory all describe the same entry. using var output = new MemoryStream(); using ( var writer = new ZipWriter( @@ -56,22 +148,150 @@ public class Zip64DescriptorSizeTests ) ) { - using var first = new MemoryStream([42]); - writer.Write("first", first, DateTime.Now); + using (var first = writer.WriteToStream("first", new ZipWriterEntryOptions())) + { + var buffer = new byte[8192]; + for (var remaining = size; remaining > 0; ) + { + var count = Math.Min(remaining, buffer.Length); + first.Write(buffer, 0, count); + remaining -= count; + } + } using var second = new MemoryStream([43]); writer.Write("second", second, DateTime.Now); } + var zip = output.ToArray(); - var descriptor = Enumerable - .Range(0, zip.Length - 24) - .First(i => BinaryPrimitives.ReadUInt32LittleEndian(zip.AsSpan(i)) == 0x08074b50); - // Parser-only metadata probe: advertise a large size without allocating a large payload. - // The Archive API is intentionally not used: this synthetic descriptor differs from the directory. - BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), size); + if (!signature) + { + var descriptor = FindDescriptor(zip); + zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray(); + } + var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync); + Assert.Equal(size, firstSize); + Assert.Equal("second", secondKey); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Classic_Descriptor_With_High_Version_Is_Read(bool useAsync, bool signature) + { + var zip = WriteZip64Archive(useZip64: false); + var descriptor = FindDescriptor(zip); + // Higher versions are also used by non-ZIP64 compression methods. Version alone + // must not select 64-bit descriptor sizes. + BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(4), 63); if (!signature) { zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray(); } + + var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync); + Assert.Equal(1L, firstSize); + Assert.Equal("second", secondKey); + } + + public static IEnumerable CompressedSizes() + { + foreach (var useAsync in new[] { false, true }) + { + foreach (var signature in new[] { false, true }) + { + foreach (var zip64Header in new[] { false, true }) + { + foreach (var size in new[] { 0xFFFFFFFFUL, 0x1FFFFFFFFUL }) + { + yield return new object[] { useAsync, signature, zip64Header, size }; + } + } + } + } + } + + [Theory] + [MemberData(nameof(CompressedSizes))] + public async Task Descriptor_Preserves_Compressed_Size_Words( + bool useAsync, + bool signature, + bool zip64Header, + ulong size + ) + { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + // Only the descriptor is changed: decompression still consumes the tiny payload. + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 8), size); + if (!zip64Header) + { + // Preserve support for older writers that emitted 64-bit descriptors without + // declaring ZIP64 in the local header. + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0); + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0); + var extra = 30 + BinaryPrimitives.ReadUInt16LittleEndian(zip.AsSpan(26)); + BinaryPrimitives.WriteUInt16LittleEndian(zip.AsSpan(extra), 0xFFFF); + } + if (!signature) + { + zip = zip.Take(descriptor).Concat(zip.Skip(descriptor + 4)).ToArray(); + } + + await ReadFirstTwoEntries(zip, useAsync, expectedCompressedSize: (long)size); + } + + /// + /// The descriptor width is also resolved when the archive is read from a seekable + /// stream, where the entry data position is calculated from the descriptor sizes. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Descriptor_Size_Collision_Is_Resolved_On_Seekable_Stream(bool useAsync) + { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), 0x04034B50UL); + + var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync, seekable: true); + Assert.Equal(0x04034B50L, firstSize); + Assert.Equal("second", secondKey); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Zip64_Extra_Field_Declares_Width_With_Zero_Local_Sizes(bool useAsync) + { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(18), 0); + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(22), 0); + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), 0x02014B50UL); + + var (firstSize, secondKey) = await ReadFirstTwoEntries(zip, useAsync); + Assert.Equal(0x02014B50L, firstSize); + Assert.Equal("second", secondKey); + } + + /// + /// A record with an unknown signature after a ZIP64 descriptor ends the enumeration, + /// but the sizes of the entry that was read are kept. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Unknown_Record_After_Zip64_Descriptor_Keeps_Sizes(bool useAsync) + { + var zip = WriteZip64Archive(); + var descriptor = FindDescriptor(zip); + const long size = 0x04034B50; + BinaryPrimitives.WriteUInt64LittleEndian(zip.AsSpan(descriptor + 16), (ulong)size); + // Corrupt the signature of the local header that follows the first descriptor. + BinaryPrimitives.WriteUInt32LittleEndian(zip.AsSpan(descriptor + 24), 0x04034B51); + using var input = new ForwardOnlyStream(new MemoryStream(zip)); if (useAsync) { @@ -82,6 +302,67 @@ public class Zip64DescriptorSizeTests ); Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None)); var first = reader.Entry; + Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None)); + Assert.Equal(size, first.Size); + } + else + { + using var reader = ReaderFactory.OpenReader(input, ReaderOptions.ForExternalStream); + Assert.True(reader.MoveToNextEntry()); + var first = reader.Entry; + Assert.False(reader.MoveToNextEntry()); + Assert.Equal(size, first.Size); + } + } + + /// + /// Writes two entries to a forward-only stream, which makes the writer use data + /// descriptors and reserve the ZIP64 fields in the local headers. + /// + private static byte[] WriteZip64Archive(bool useZip64 = true) + { + using var output = new MemoryStream(); + using ( + var writer = new ZipWriter( + new ForwardOnlyStream(output), + new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = useZip64 } + ) + ) + { + using var first = new MemoryStream([42]); + writer.Write("first", first, DateTime.Now); + using var second = new MemoryStream([43]); + writer.Write("second", second, DateTime.Now); + } + return output.ToArray(); + } + + private static int FindDescriptor(byte[] zip) => + Enumerable + .Range(0, zip.Length - 24) + .First(i => + BinaryPrimitives.ReadUInt32LittleEndian(zip.AsSpan(i)) == DataDescriptorSignature + ); + + private static async Task<(long FirstSize, string? SecondKey)> ReadFirstTwoEntries( + byte[] zip, + bool useAsync, + bool seekable = false, + long? expectedCompressedSize = null + ) + { + using var input = seekable + ? (Stream)new MemoryStream(zip) + : new ForwardOnlyStream(new MemoryStream(zip)); + if (useAsync) + { + await using var reader = await ReaderFactory.OpenAsyncReader( + seekable ? input : new AsyncOnlyStream(input), + ReaderOptions.ForExternalStream, + CancellationToken.None + ); + Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None)); + var first = reader.Entry; #if LEGACY_DOTNET using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None)) #else @@ -91,8 +372,24 @@ public class Zip64DescriptorSizeTests await stream.CopyToAsync(Stream.Null, 81920, CancellationToken.None); } Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None)); - Assert.Equal((long)size, first.Size); + if (expectedCompressedSize.HasValue) + { + Assert.Equal(expectedCompressedSize.Value, first.CompressedSize); + } Assert.Equal("second", reader.Entry.Key); + using var second = new MemoryStream(); +#if LEGACY_DOTNET + using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None)) +#else + await using (var stream = await reader.OpenEntryStreamAsync(CancellationToken.None)) +#endif + { + await stream.CopyToAsync(second, 81920, CancellationToken.None); + } + Assert.Equal(new byte[] { 43 }, second.ToArray()); + var secondKey = reader.Entry.Key; + Assert.False(await reader.MoveToNextEntryAsync(CancellationToken.None)); + return (first.Size, secondKey); } else { @@ -104,8 +401,20 @@ public class Zip64DescriptorSizeTests stream.CopyTo(Stream.Null); } Assert.True(reader.MoveToNextEntry()); - Assert.Equal((long)size, first.Size); + if (expectedCompressedSize.HasValue) + { + Assert.Equal(expectedCompressedSize.Value, first.CompressedSize); + } Assert.Equal("second", reader.Entry.Key); + using var second = new MemoryStream(); + using (var stream = reader.OpenEntryStream()) + { + stream.CopyTo(second); + } + Assert.Equal(new byte[] { 43 }, second.ToArray()); + var secondKey = reader.Entry.Key; + Assert.False(reader.MoveToNextEntry()); + return (first.Size, secondKey); } } }