mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-07-08 18:16:30 +00:00
Merge pull request #1362 from LANCommander/fix-zip64-streaming-corrupted-entry-sizes
Zip: fix Zip64 streaming reader corrupting entry size/CRC and failing on non-seekable streams
This commit is contained in:
Adam Hathcock
GitHub
@@ -189,64 +189,77 @@ internal sealed partial class StreamingZipHeaderFactory
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime
|
||||
|
||||
var crc = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (crc == POST_DATA_DESCRIPTOR)
|
||||
// A Zip64 entry that does not use a post-data descriptor stores its real CRC
|
||||
// and sizes in the local header's Zip64 extra field, so a header signature
|
||||
// (the next local entry, or the central directory) follows the data directly.
|
||||
// In that case the entry's metadata is already correct and must not be
|
||||
// overwritten with bytes read from the following header. We have only consumed
|
||||
// the 4-byte signature, so fall through and parse this header normally. Because
|
||||
// no seek-back is required here, this also works for non-seekable streams.
|
||||
if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50)
|
||||
{
|
||||
crc = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
lastEntryHeader.Crc = crc;
|
||||
lastEntryHeader.IsCrcAvailable = true;
|
||||
|
||||
// The DataDescriptor can be either 64bit or 32bit
|
||||
var compressedSize = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var uncompressedSize = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Check if we have header or 64bit DataDescriptor
|
||||
var testHeader = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50);
|
||||
|
||||
var test64Bit = ((long)uncompressedSize << 32) | compressedSize;
|
||||
if (test64Bit == lastEntryHeader.CompressedSize && testHeader)
|
||||
{
|
||||
lastEntryHeader.UncompressedSize =
|
||||
(
|
||||
(long)
|
||||
await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false) << 32
|
||||
) | headerBytes;
|
||||
headerBytes = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (pos.HasValue)
|
||||
{
|
||||
lastEntryHeader.DataStartPosition =
|
||||
pos - lastEntryHeader.CompressedSize;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
lastEntryHeader.UncompressedSize = uncompressedSize;
|
||||
}
|
||||
// A data descriptor follows. Recover the CRC and sizes from it; the
|
||||
// descriptor can carry either 32-bit or 64-bit sizes.
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate
|
||||
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime
|
||||
|
||||
if (pos.HasValue)
|
||||
{
|
||||
lastEntryHeader.DataStartPosition = pos - lastEntryHeader.CompressedSize;
|
||||
var crc = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// For SeekableSharpCompressStream, seek back to just after the local header signature.
|
||||
// Plain SharpCompressStream cannot seek to arbitrary positions, so we skip this.
|
||||
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
|
||||
if (_sharpCompressStream is SeekableSharpCompressStream)
|
||||
if (crc == POST_DATA_DESCRIPTOR)
|
||||
{
|
||||
crc = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
lastEntryHeader.Crc = crc;
|
||||
lastEntryHeader.IsCrcAvailable = true;
|
||||
|
||||
// The DataDescriptor can be either 64bit or 32bit
|
||||
var compressedSize = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var uncompressedSize = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var test64Bit = ((long)uncompressedSize << 32) | compressedSize;
|
||||
if (test64Bit == lastEntryHeader.CompressedSize)
|
||||
{
|
||||
lastEntryHeader.UncompressedSize =
|
||||
(
|
||||
(long)
|
||||
await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false) << 32
|
||||
) | headerBytes;
|
||||
headerBytes = await _reader
|
||||
.ReadUInt32Async(_cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
lastEntryHeader.UncompressedSize = uncompressedSize;
|
||||
}
|
||||
|
||||
if (pos.HasValue)
|
||||
{
|
||||
lastEntryHeader.DataStartPosition =
|
||||
pos - lastEntryHeader.CompressedSize;
|
||||
|
||||
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
|
||||
_sharpCompressStream.Position = pos.Value + 4;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,46 +109,63 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory
|
||||
|
||||
headerBytes = reader.ReadUInt32();
|
||||
|
||||
var version = reader.ReadUInt16();
|
||||
var flags = (HeaderFlags)reader.ReadUInt16();
|
||||
var compressionMethod = (ZipCompressionMethod)reader.ReadUInt16();
|
||||
var lastModifiedDate = reader.ReadUInt16();
|
||||
var lastModifiedTime = reader.ReadUInt16();
|
||||
|
||||
var crc = reader.ReadUInt32();
|
||||
|
||||
if (crc == POST_DATA_DESCRIPTOR)
|
||||
// A Zip64 entry that does not use a post-data descriptor stores its real CRC
|
||||
// and sizes in the local header's Zip64 extra field, so a header signature
|
||||
// (the next local entry, or the central directory) follows the data directly.
|
||||
// In that case the entry's metadata is already correct and must not be
|
||||
// overwritten with bytes read from the following header. We have only consumed
|
||||
// the 4-byte signature, so fall through and parse this header normally.
|
||||
if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50)
|
||||
{
|
||||
crc = reader.ReadUInt32();
|
||||
}
|
||||
_lastEntryHeader.Crc = crc;
|
||||
_lastEntryHeader.IsCrcAvailable = true;
|
||||
|
||||
// The DataDescriptor can be either 64bit or 32bit
|
||||
var compressed_size = reader.ReadUInt32();
|
||||
var uncompressed_size = reader.ReadUInt32();
|
||||
|
||||
// Check if we have header or 64bit DataDescriptor
|
||||
var test_header = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50);
|
||||
|
||||
var test_64bit = ((long)uncompressed_size << 32) | compressed_size;
|
||||
if (test_64bit == _lastEntryHeader.CompressedSize && test_header)
|
||||
{
|
||||
_lastEntryHeader.UncompressedSize =
|
||||
((long)reader.ReadUInt32() << 32) | headerBytes;
|
||||
headerBytes = reader.ReadUInt32();
|
||||
if (pos.HasValue)
|
||||
{
|
||||
_lastEntryHeader.DataStartPosition =
|
||||
pos - _lastEntryHeader.CompressedSize;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_lastEntryHeader.UncompressedSize = uncompressed_size;
|
||||
}
|
||||
// A data descriptor follows. Recover the CRC and sizes from it; the
|
||||
// descriptor can carry either 32-bit or 64-bit sizes.
|
||||
_ = reader.ReadUInt16(); // version
|
||||
_ = reader.ReadUInt16(); // flags
|
||||
_ = reader.ReadUInt16(); // compressionMethod
|
||||
_ = reader.ReadUInt16(); // lastModifiedDate
|
||||
_ = reader.ReadUInt16(); // lastModifiedTime
|
||||
|
||||
if (pos.HasValue)
|
||||
{
|
||||
_lastEntryHeader.DataStartPosition = pos - _lastEntryHeader.CompressedSize;
|
||||
var crc = reader.ReadUInt32();
|
||||
|
||||
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
|
||||
sharpCompressStream.Position = pos.Value + 4;
|
||||
if (crc == POST_DATA_DESCRIPTOR)
|
||||
{
|
||||
crc = reader.ReadUInt32();
|
||||
}
|
||||
_lastEntryHeader.Crc = crc;
|
||||
_lastEntryHeader.IsCrcAvailable = true;
|
||||
|
||||
// The DataDescriptor can be either 64bit or 32bit
|
||||
var compressed_size = reader.ReadUInt32();
|
||||
var uncompressed_size = reader.ReadUInt32();
|
||||
|
||||
var test_64bit = ((long)uncompressed_size << 32) | compressed_size;
|
||||
if (test_64bit == _lastEntryHeader.CompressedSize)
|
||||
{
|
||||
_lastEntryHeader.UncompressedSize =
|
||||
((long)reader.ReadUInt32() << 32) | headerBytes;
|
||||
headerBytes = reader.ReadUInt32();
|
||||
}
|
||||
else
|
||||
{
|
||||
_lastEntryHeader.UncompressedSize = uncompressed_size;
|
||||
}
|
||||
|
||||
if (pos.HasValue)
|
||||
{
|
||||
_lastEntryHeader.DataStartPosition =
|
||||
pos - _lastEntryHeader.CompressedSize;
|
||||
|
||||
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
|
||||
sharpCompressStream.Position = pos.Value + 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
@@ -89,6 +89,135 @@ public class Zip64AsyncTests : WriterTests
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
// Regression test for reading a Zip64 archive over a *non-seekable* async stream, as
|
||||
// happens when extracting directly from a network download. When a >=4GB (Zip64) entry
|
||||
// is followed by another entry, the streaming reader probes a few bytes past the big
|
||||
// entry's data to locate the next header and must rewind them. For non-seekable streams
|
||||
// it previously failed to do so (the rewind was gated on SeekableSharpCompressStream),
|
||||
// leaving the reader misaligned so the *following* local header was parsed from garbage
|
||||
// and extraction threw near the very end. A seekable stream rewinds correctly and works,
|
||||
// which is exactly the "works seekable, fails non-seekable" symptom that was reported.
|
||||
//
|
||||
// NOTE: heavy (~4GB) like the other Zip64 large-file tests in this file, hence disabled
|
||||
// by default. Enable to verify the fix.
|
||||
//[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async ValueTask Zip64_Large_File_Then_Small_File_NonSeekable_Async()
|
||||
{
|
||||
var filename = Path.Combine(SCRATCH2_FILES_PATH, "zip64-nonseekable-async.zip");
|
||||
|
||||
// A small trailing entry with recognizable content. Its bytes can only be read back
|
||||
// correctly if the reader stays byte-aligned after the preceding >=4GB Zip64 entry.
|
||||
var smallContent = new byte[64 * 1024];
|
||||
for (var i = 0; i < smallContent.Length; i++)
|
||||
{
|
||||
smallContent[i] = (byte)(i % 251);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
File.Delete(filename);
|
||||
}
|
||||
|
||||
CreateLargeThenSmallZip(filename, FOUR_GB_LIMIT, smallContent);
|
||||
|
||||
var (count, lastKey, lastContent) = await ReadLargeThenSmallNonSeekableAsync(filename);
|
||||
|
||||
// The reader must reach the second (small) entry without throwing, identify it
|
||||
// correctly, and read its bytes verbatim.
|
||||
Assert.Equal(2, count);
|
||||
Assert.Equal("small", lastKey);
|
||||
Assert.NotNull(lastContent);
|
||||
Assert.Equal(smallContent, lastContent!);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
File.Delete(filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void CreateLargeThenSmallZip(string filename, long largeSize, byte[] smallContent)
|
||||
{
|
||||
var chunk = new byte[1024 * 1024];
|
||||
|
||||
// Force Zip64 and store (level 0) so the large entry's compressed size also exceeds
|
||||
// 4GiB, which is what marks the entry as Zip64 for the streaming reader.
|
||||
var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = true };
|
||||
var eo = new ZipWriterEntryOptions { CompressionLevel = 0 };
|
||||
|
||||
using var zip = File.OpenWrite(filename);
|
||||
using var zipWriter = (ZipWriter)WriterFactory.OpenWriter(zip, ArchiveType.Zip, opts);
|
||||
|
||||
using (var str = zipWriter.WriteToStream("large", eo))
|
||||
{
|
||||
var left = largeSize;
|
||||
while (left > 0)
|
||||
{
|
||||
var b = (int)Math.Min(left, chunk.Length);
|
||||
str.Write(chunk, 0, b);
|
||||
left -= b;
|
||||
}
|
||||
}
|
||||
|
||||
using (var str = zipWriter.WriteToStream("small", eo))
|
||||
{
|
||||
str.Write(smallContent, 0, smallContent.Length);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<(
|
||||
long Count,
|
||||
string? LastKey,
|
||||
byte[]? LastContent
|
||||
)> ReadLargeThenSmallNonSeekableAsync(string filename)
|
||||
{
|
||||
long count = 0;
|
||||
string? lastKey = null;
|
||||
byte[]? lastContent = null;
|
||||
|
||||
using var fs = File.OpenRead(filename);
|
||||
// ForwardOnlyStream reports CanSeek == false; AsyncOnlyStream forces async reads.
|
||||
// Together they emulate a non-seekable, async-only source (e.g. a network download).
|
||||
//
|
||||
// IMPORTANT: use default ReaderOptions (LeaveStreamOpen == false), exactly as the
|
||||
// reporting user did. With LeaveStreamOpen == true the Volume wraps the stream in a
|
||||
// passthrough that Create() later unwraps into a SeekableSharpCompressStream, which
|
||||
// happens to take the working seek-back path and hides the bug. The default keeps a
|
||||
// plain ring-buffer SharpCompressStream, which is where the streaming reader fails.
|
||||
await using var rd = await ReaderFactory.OpenAsyncReader(
|
||||
new AsyncOnlyStream(new ForwardOnlyStream(fs)),
|
||||
new ReaderOptions { LookForHeader = false }
|
||||
);
|
||||
while (await rd.MoveToNextEntryAsync())
|
||||
{
|
||||
count++;
|
||||
lastKey = rd.Entry.Key;
|
||||
|
||||
#if LEGACY_DOTNET
|
||||
using var entryStream = await rd.OpenEntryStreamAsync();
|
||||
#else
|
||||
await using var entryStream = await rd.OpenEntryStreamAsync();
|
||||
#endif
|
||||
if (rd.Entry.Key == "small")
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
await entryStream.CopyToAsync(ms);
|
||||
lastContent = ms.ToArray();
|
||||
}
|
||||
else
|
||||
{
|
||||
await entryStream.SkipEntryAsync();
|
||||
}
|
||||
}
|
||||
|
||||
return (count, lastKey, lastContent);
|
||||
}
|
||||
|
||||
public async ValueTask RunSingleTestAsync(
|
||||
long files,
|
||||
long filesize,
|
||||
|
||||
Reference in New Issue
Block a user