diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs index 4e3bcc3a..f6ddfb2c 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs @@ -189,64 +189,77 @@ internal sealed partial class StreamingZipHeaderFactory .ReadUInt32Async(_cancellationToken) .ConfigureAwait(false); - _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version - _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags - _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod - _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate - _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime - - var crc = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - - if (crc == POST_DATA_DESCRIPTOR) + // A Zip64 entry that does not use a post-data descriptor stores its real CRC + // and sizes in the local header's Zip64 extra field, so a header signature + // (the next local entry, or the central directory) follows the data directly. + // In that case the entry's metadata is already correct and must not be + // overwritten with bytes read from the following header. We have only consumed + // the 4-byte signature, so fall through and parse this header normally. Because + // no seek-back is required here, this also works for non-seekable streams. + if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50) { - crc = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - } - lastEntryHeader.Crc = crc; - lastEntryHeader.IsCrcAvailable = true; - - // The DataDescriptor can be either 64bit or 32bit - var compressedSize = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - var uncompressedSize = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); - - // Check if we have header or 64bit DataDescriptor - var testHeader = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50); - - var test64Bit = ((long)uncompressedSize << 32) | compressedSize; - if (test64Bit == lastEntryHeader.CompressedSize && testHeader) - { - lastEntryHeader.UncompressedSize = - ( - (long) - await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false) << 32 - ) | headerBytes; - headerBytes = await _reader - .ReadUInt32Async(_cancellationToken) - .ConfigureAwait(false); + if (pos.HasValue) + { + lastEntryHeader.DataStartPosition = + pos - lastEntryHeader.CompressedSize; + } } else { - lastEntryHeader.UncompressedSize = uncompressedSize; - } + // A data descriptor follows. Recover the CRC and sizes from it; the + // descriptor can carry either 32-bit or 64-bit sizes. + _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version + _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags + _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod + _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate + _ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime - if (pos.HasValue) - { - lastEntryHeader.DataStartPosition = pos - lastEntryHeader.CompressedSize; + var crc = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); - // For SeekableSharpCompressStream, seek back to just after the local header signature. - // Plain SharpCompressStream cannot seek to arbitrary positions, so we skip this. - // 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04) - if (_sharpCompressStream is SeekableSharpCompressStream) + if (crc == POST_DATA_DESCRIPTOR) { + crc = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + } + lastEntryHeader.Crc = crc; + lastEntryHeader.IsCrcAvailable = true; + + // The DataDescriptor can be either 64bit or 32bit + var compressedSize = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + var uncompressedSize = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + + var test64Bit = ((long)uncompressedSize << 32) | compressedSize; + if (test64Bit == lastEntryHeader.CompressedSize) + { + lastEntryHeader.UncompressedSize = + ( + (long) + await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false) << 32 + ) | headerBytes; + headerBytes = await _reader + .ReadUInt32Async(_cancellationToken) + .ConfigureAwait(false); + } + else + { + lastEntryHeader.UncompressedSize = uncompressedSize; + } + + if (pos.HasValue) + { + lastEntryHeader.DataStartPosition = + pos - lastEntryHeader.CompressedSize; + + // 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04) _sharpCompressStream.Position = pos.Value + 4; } } diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs index c955afb0..8e775727 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs @@ -109,46 +109,63 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory headerBytes = reader.ReadUInt32(); - var version = reader.ReadUInt16(); - var flags = (HeaderFlags)reader.ReadUInt16(); - var compressionMethod = (ZipCompressionMethod)reader.ReadUInt16(); - var lastModifiedDate = reader.ReadUInt16(); - var lastModifiedTime = reader.ReadUInt16(); - - var crc = reader.ReadUInt32(); - - if (crc == POST_DATA_DESCRIPTOR) + // A Zip64 entry that does not use a post-data descriptor stores its real CRC + // and sizes in the local header's Zip64 extra field, so a header signature + // (the next local entry, or the central directory) follows the data directly. + // In that case the entry's metadata is already correct and must not be + // overwritten with bytes read from the following header. We have only consumed + // the 4-byte signature, so fall through and parse this header normally. + if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50) { - crc = reader.ReadUInt32(); - } - _lastEntryHeader.Crc = crc; - _lastEntryHeader.IsCrcAvailable = true; - - // The DataDescriptor can be either 64bit or 32bit - var compressed_size = reader.ReadUInt32(); - var uncompressed_size = reader.ReadUInt32(); - - // Check if we have header or 64bit DataDescriptor - var test_header = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50); - - var test_64bit = ((long)uncompressed_size << 32) | compressed_size; - if (test_64bit == _lastEntryHeader.CompressedSize && test_header) - { - _lastEntryHeader.UncompressedSize = - ((long)reader.ReadUInt32() << 32) | headerBytes; - headerBytes = reader.ReadUInt32(); + if (pos.HasValue) + { + _lastEntryHeader.DataStartPosition = + pos - _lastEntryHeader.CompressedSize; + } } else { - _lastEntryHeader.UncompressedSize = uncompressed_size; - } + // A data descriptor follows. Recover the CRC and sizes from it; the + // descriptor can carry either 32-bit or 64-bit sizes. + _ = reader.ReadUInt16(); // version + _ = reader.ReadUInt16(); // flags + _ = reader.ReadUInt16(); // compressionMethod + _ = reader.ReadUInt16(); // lastModifiedDate + _ = reader.ReadUInt16(); // lastModifiedTime - if (pos.HasValue) - { - _lastEntryHeader.DataStartPosition = pos - _lastEntryHeader.CompressedSize; + var crc = reader.ReadUInt32(); - // 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04) - sharpCompressStream.Position = pos.Value + 4; + if (crc == POST_DATA_DESCRIPTOR) + { + crc = reader.ReadUInt32(); + } + _lastEntryHeader.Crc = crc; + _lastEntryHeader.IsCrcAvailable = true; + + // The DataDescriptor can be either 64bit or 32bit + var compressed_size = reader.ReadUInt32(); + var uncompressed_size = reader.ReadUInt32(); + + var test_64bit = ((long)uncompressed_size << 32) | compressed_size; + if (test_64bit == _lastEntryHeader.CompressedSize) + { + _lastEntryHeader.UncompressedSize = + ((long)reader.ReadUInt32() << 32) | headerBytes; + headerBytes = reader.ReadUInt32(); + } + else + { + _lastEntryHeader.UncompressedSize = uncompressed_size; + } + + if (pos.HasValue) + { + _lastEntryHeader.DataStartPosition = + pos - _lastEntryHeader.CompressedSize; + + // 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04) + sharpCompressStream.Position = pos.Value + 4; + } } } else diff --git a/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs b/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs index 32e666b4..5124b6b0 100644 --- a/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs +++ b/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs @@ -89,6 +89,135 @@ public class Zip64AsyncTests : WriterTests catch (NotSupportedException) { } } + // Regression test for reading a Zip64 archive over a *non-seekable* async stream, as + // happens when extracting directly from a network download. When a >=4GB (Zip64) entry + // is followed by another entry, the streaming reader probes a few bytes past the big + // entry's data to locate the next header and must rewind them. For non-seekable streams + // it previously failed to do so (the rewind was gated on SeekableSharpCompressStream), + // leaving the reader misaligned so the *following* local header was parsed from garbage + // and extraction threw near the very end. A seekable stream rewinds correctly and works, + // which is exactly the "works seekable, fails non-seekable" symptom that was reported. + // + // NOTE: heavy (~4GB) like the other Zip64 large-file tests in this file, hence disabled + // by default. Enable to verify the fix. + //[Fact] + [Trait("format", "zip64")] + public async ValueTask Zip64_Large_File_Then_Small_File_NonSeekable_Async() + { + var filename = Path.Combine(SCRATCH2_FILES_PATH, "zip64-nonseekable-async.zip"); + + // A small trailing entry with recognizable content. Its bytes can only be read back + // correctly if the reader stays byte-aligned after the preceding >=4GB Zip64 entry. + var smallContent = new byte[64 * 1024]; + for (var i = 0; i < smallContent.Length; i++) + { + smallContent[i] = (byte)(i % 251); + } + + try + { + if (File.Exists(filename)) + { + File.Delete(filename); + } + + CreateLargeThenSmallZip(filename, FOUR_GB_LIMIT, smallContent); + + var (count, lastKey, lastContent) = await ReadLargeThenSmallNonSeekableAsync(filename); + + // The reader must reach the second (small) entry without throwing, identify it + // correctly, and read its bytes verbatim. + Assert.Equal(2, count); + Assert.Equal("small", lastKey); + Assert.NotNull(lastContent); + Assert.Equal(smallContent, lastContent!); + } + finally + { + if (File.Exists(filename)) + { + File.Delete(filename); + } + } + } + + private void CreateLargeThenSmallZip(string filename, long largeSize, byte[] smallContent) + { + var chunk = new byte[1024 * 1024]; + + // Force Zip64 and store (level 0) so the large entry's compressed size also exceeds + // 4GiB, which is what marks the entry as Zip64 for the streaming reader. + var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = true }; + var eo = new ZipWriterEntryOptions { CompressionLevel = 0 }; + + using var zip = File.OpenWrite(filename); + using var zipWriter = (ZipWriter)WriterFactory.OpenWriter(zip, ArchiveType.Zip, opts); + + using (var str = zipWriter.WriteToStream("large", eo)) + { + var left = largeSize; + while (left > 0) + { + var b = (int)Math.Min(left, chunk.Length); + str.Write(chunk, 0, b); + left -= b; + } + } + + using (var str = zipWriter.WriteToStream("small", eo)) + { + str.Write(smallContent, 0, smallContent.Length); + } + } + + private async ValueTask<( + long Count, + string? LastKey, + byte[]? LastContent + )> ReadLargeThenSmallNonSeekableAsync(string filename) + { + long count = 0; + string? lastKey = null; + byte[]? lastContent = null; + + using var fs = File.OpenRead(filename); + // ForwardOnlyStream reports CanSeek == false; AsyncOnlyStream forces async reads. + // Together they emulate a non-seekable, async-only source (e.g. a network download). + // + // IMPORTANT: use default ReaderOptions (LeaveStreamOpen == false), exactly as the + // reporting user did. With LeaveStreamOpen == true the Volume wraps the stream in a + // passthrough that Create() later unwraps into a SeekableSharpCompressStream, which + // happens to take the working seek-back path and hides the bug. The default keeps a + // plain ring-buffer SharpCompressStream, which is where the streaming reader fails. + await using var rd = await ReaderFactory.OpenAsyncReader( + new AsyncOnlyStream(new ForwardOnlyStream(fs)), + new ReaderOptions { LookForHeader = false } + ); + while (await rd.MoveToNextEntryAsync()) + { + count++; + lastKey = rd.Entry.Key; + +#if LEGACY_DOTNET + using var entryStream = await rd.OpenEntryStreamAsync(); +#else + await using var entryStream = await rd.OpenEntryStreamAsync(); +#endif + if (rd.Entry.Key == "small") + { + using var ms = new MemoryStream(); + await entryStream.CopyToAsync(ms); + lastContent = ms.ToArray(); + } + else + { + await entryStream.SkipEntryAsync(); + } + } + + return (count, lastKey, lastContent); + } + public async ValueTask RunSingleTestAsync( long files, long filesize,