starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-23 15:34:34 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Pass more Zip tests

Browse Source
This commit is contained in:
Adam Hathcock
2026-01-07 14:47:20 +00:00
parent 39d85ff4f6
commit c3fd42057a
6 changed files with 632 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/SharpCompress/Archives/Zip/ZipArchive.cs
View File

@@ -299,9 +299,10 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
stream = new SharpCompressStream(stream, bufferSize: bufferSize);
}
var header = headerFactory
.ReadStreamHeader(stream)
.FirstOrDefault(x => x.ZipHeaderType != ZipHeaderType.Split);
var header = await headerFactory
.ReadStreamHeaderAsync(stream)
.Where(x => x.ZipHeaderType != ZipHeaderType.Split)
.FirstOrDefaultAsync();
if (header is null)
{
return false;

330
src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs
View File

@@ -2,6 +2,9 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
@@ -200,4 +203,331 @@ internal class StreamingZipHeaderFactory : ZipHeaderFactory
yield return header;
}
}
/// <summary>
/// Reads ZIP headers asynchronously for streams that do not support synchronous reads.
/// </summary>
internal IAsyncEnumerable<ZipHeader> ReadStreamHeaderAsync(Stream stream) =>
new StreamHeaderAsyncEnumerable(this, stream);
/// <summary>
/// Invokes the shared async header parsing logic on the base factory.
/// </summary>
private ValueTask<ZipHeader?> ReadHeaderAsyncInternal(
uint headerBytes,
AsyncBinaryReader reader
) => ReadHeader(headerBytes, reader);
/// <summary>
/// Exposes the last parsed local entry header to the async enumerator so it can handle streaming data descriptors.
/// </summary>
private LocalEntryHeader? LastEntryHeader
{
get => _lastEntryHeader;
set => _lastEntryHeader = value;
}
/// <summary>
/// Produces an async enumerator for streaming ZIP headers.
/// </summary>
private sealed class StreamHeaderAsyncEnumerable : IAsyncEnumerable<ZipHeader>
{
private readonly StreamingZipHeaderFactory _headerFactory;
private readonly Stream _stream;
public StreamHeaderAsyncEnumerable(StreamingZipHeaderFactory headerFactory, Stream stream)
{
_headerFactory = headerFactory;
_stream = stream;
}
public IAsyncEnumerator<ZipHeader> GetAsyncEnumerator(
CancellationToken cancellationToken = default
) => new StreamHeaderAsyncEnumerator(_headerFactory, _stream, cancellationToken);
}
/// <summary>
/// Async implementation of <see cref="ReadStreamHeader"/> using <see cref="AsyncBinaryReader"/> to avoid sync reads.
/// </summary>
private sealed class StreamHeaderAsyncEnumerator : IAsyncEnumerator<ZipHeader>, IDisposable
{
private readonly StreamingZipHeaderFactory _headerFactory;
private readonly SharpCompressStream _rewindableStream;
private readonly AsyncBinaryReader _reader;
private readonly CancellationToken _cancellationToken;
private bool _completed;
public StreamHeaderAsyncEnumerator(
StreamingZipHeaderFactory headerFactory,
Stream stream,
CancellationToken cancellationToken
)
{
_headerFactory = headerFactory;
_rewindableStream = EnsureSharpCompressStream(stream);
_reader = new AsyncBinaryReader(_rewindableStream, leaveOpen: true);
_cancellationToken = cancellationToken;
}
private ZipHeader? _current;
public ZipHeader Current =>
_current ?? throw new InvalidOperationException("No current header is available.");
/// <summary>
/// Advances to the next ZIP header in the stream, honoring streaming data descriptors where applicable.
/// </summary>
public async ValueTask<bool> MoveNextAsync()
{
if (_completed)
{
return false;
}
while (true)
{
_cancellationToken.ThrowIfCancellationRequested();
uint headerBytes;
var lastEntryHeader = _headerFactory.LastEntryHeader;
if (
lastEntryHeader != null
&& FlagUtility.HasFlag(lastEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor)
)
{
if (lastEntryHeader.Part is null)
{
continue;
}
var pos = _rewindableStream.CanSeek ? (long?)_rewindableStream.Position : null;
var crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
if (crc == POST_DATA_DESCRIPTOR)
{
crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
lastEntryHeader.Crc = crc;
//attempt 32bit read
ulong compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
ulong uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
//check for zip64 sentinel or unexpected header
bool isSentinel =
compressedSize == 0xFFFFFFFF || uncompressedSize == 0xFFFFFFFF;
bool isHeader = headerBytes == 0x04034b50 || headerBytes == 0x02014b50;
if (!isHeader && !isSentinel)
{
//reshuffle into 64-bit values
compressedSize = (uncompressedSize << 32) | compressedSize;
uncompressedSize =
((ulong)headerBytes << 32)
| await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else if (isSentinel)
{
//standards-compliant zip64 descriptor
compressedSize = await _reader
.ReadUInt64Async(_cancellationToken)
.ConfigureAwait(false);
uncompressedSize = await _reader
.ReadUInt64Async(_cancellationToken)
.ConfigureAwait(false);
}
lastEntryHeader.CompressedSize = (long)compressedSize;
lastEntryHeader.UncompressedSize = (long)uncompressedSize;
if (pos.HasValue)
{
lastEntryHeader.DataStartPosition = pos - lastEntryHeader.CompressedSize;
}
}
else if (lastEntryHeader != null && lastEntryHeader.IsZip64)
{
if (lastEntryHeader.Part is null)
{
continue;
}
var pos = _rewindableStream.CanSeek ? (long?)_rewindableStream.Position : null;
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime
var crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
if (crc == POST_DATA_DESCRIPTOR)
{
crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
lastEntryHeader.Crc = crc;
// The DataDescriptor can be either 64bit or 32bit
var compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
var uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
// Check if we have header or 64bit DataDescriptor
var testHeader = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50);
var test64Bit = ((long)uncompressedSize << 32) | compressedSize;
if (test64Bit == lastEntryHeader.CompressedSize && testHeader)
{
lastEntryHeader.UncompressedSize =
(
(long)
await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false) << 32
) | headerBytes;
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else
{
lastEntryHeader.UncompressedSize = uncompressedSize;
}
if (pos.HasValue)
{
lastEntryHeader.DataStartPosition = pos - lastEntryHeader.CompressedSize;
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
_rewindableStream.Position = pos.Value + 4;
}
}
else
{
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
_headerFactory.LastEntryHeader = null;
var header = await _headerFactory
.ReadHeaderAsyncInternal(headerBytes, _reader)
.ConfigureAwait(false);
if (header is null)
{
_completed = true;
return false;
}
//entry could be zero bytes so we need to know that.
if (header.ZipHeaderType == ZipHeaderType.LocalEntry)
{
var localHeader = (LocalEntryHeader)header;
var directoryHeader = _headerFactory._entries?.FirstOrDefault(entry =>
entry.Key == localHeader.Name
&& localHeader.CompressedSize == 0
&& localHeader.UncompressedSize == 0
&& localHeader.Crc == 0
&& localHeader.IsDirectory == false
);
if (directoryHeader != null)
{
localHeader.UncompressedSize = directoryHeader.Size;
localHeader.CompressedSize = directoryHeader.CompressedSize;
localHeader.Crc = (uint)directoryHeader.Crc;
}
// If we have CompressedSize, there is data to be read
if (localHeader.CompressedSize > 0)
{
header.HasData = true;
} // Check if zip is streaming ( Length is 0 and is declared in PostDataDescriptor )
else if (localHeader.Flags.HasFlag(HeaderFlags.UsePostDataDescriptor))
{
var nextHeaderBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
((IStreamStack)_rewindableStream).Rewind(sizeof(uint));
// Check if next data is PostDataDescriptor, streamed file with 0 length
header.HasData = !IsHeader(nextHeaderBytes);
}
else // We are not streaming and compressed size is 0, we have no data
{
header.HasData = false;
}
}
_current = header;
return true;
}
}
public ValueTask DisposeAsync()
{
Dispose();
return default;
}
/// <summary>
/// Disposes the underlying reader (without closing the archive stream).
/// </summary>
public void Dispose()
{
_reader.Dispose();
}
/// <summary>
/// Ensures the stream is a <see cref="SharpCompressStream"/> so header parsing can use rewind/buffer helpers.
/// </summary>
private static SharpCompressStream EnsureSharpCompressStream(Stream stream)
{
if (stream is SharpCompressStream sharpCompressStream)
{
return sharpCompressStream;
}
// Ensure the stream is already a SharpCompressStream so the buffer/size is set.
// The original code wrapped this with RewindableStream; use SharpCompressStream so we can get the buffer size.
if (stream is SourceStream src)
{
return new SharpCompressStream(
stream,
src.ReaderOptions.LeaveStreamOpen,
bufferSize: src.ReaderOptions.BufferSize
);
}
throw new ArgumentException("Stream must be a SharpCompressStream", nameof(stream));
}
}
}

81
src/SharpCompress/Common/Zip/ZipHeaderFactory.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
@@ -47,7 +48,7 @@ internal class ZipHeaderFactory
{
var entryHeader = new LocalEntryHeader(_archiveEncoding);
await entryHeader.Read(reader);
LoadHeader(entryHeader, reader.BaseStream);
await LoadHeaderAsync(entryHeader, reader.BaseStream).ConfigureAwait(false);
_lastEntryHeader = entryHeader;
return entryHeader;
@@ -282,4 +283,82 @@ internal class ZipHeaderFactory
//}
}
/// <summary>
/// Loads encryption metadata and stream positioning for a header using async reads where needed.
/// </summary>
private async ValueTask LoadHeaderAsync(ZipFileEntry entryHeader, Stream stream)
{
if (FlagUtility.HasFlag(entryHeader.Flags, HeaderFlags.Encrypted))
{
if (
!entryHeader.IsDirectory
&& entryHeader.CompressedSize == 0
&& FlagUtility.HasFlag(entryHeader.Flags, HeaderFlags.UsePostDataDescriptor)
)
{
throw new NotSupportedException(
"SharpCompress cannot currently read non-seekable Zip Streams with encrypted data that has been written in a non-seekable manner."
);
}
if (_password is null)
{
throw new CryptographicException("No password supplied for encrypted zip.");
}
entryHeader.Password = _password;
if (entryHeader.CompressionMethod == ZipCompressionMethod.WinzipAes)
{
var data = entryHeader.Extra.SingleOrDefault(x =>
x.Type == ExtraDataType.WinZipAes
);
if (data != null)
{
var keySize = (WinzipAesKeySize)data.DataBytes[4];
var salt = new byte[WinzipAesEncryptionData.KeyLengthInBytes(keySize) / 2];
var passwordVerifyValue = new byte[2];
await stream.ReadExactAsync(salt, 0, salt.Length).ConfigureAwait(false);
await stream.ReadExactAsync(passwordVerifyValue, 0, 2).ConfigureAwait(false);
entryHeader.WinzipAesEncryptionData = new WinzipAesEncryptionData(
keySize,
salt,
passwordVerifyValue,
_password
);
entryHeader.CompressedSize -= (uint)(salt.Length + 2);
}
}
}
if (entryHeader.IsDirectory)
{
return;
}
switch (_mode)
{
case StreamingMode.Seekable:
{
entryHeader.DataStartPosition = stream.Position;
stream.Position += entryHeader.CompressedSize;
break;
}
case StreamingMode.Streaming:
{
entryHeader.PackedStream = stream;
break;
}
default:
{
throw new InvalidFormatException("Invalid StreamingMode");
}
}
}
}

32
src/SharpCompress/Polyfills/AsyncEnumerableExtensions.cs Normal file
View File

@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace SharpCompress;
public static class AsyncEnumerableExtensions
{
extension<T>(IAsyncEnumerable<T> source)
{
public async IAsyncEnumerable<T> Where(Func<T, bool> predicate)
{
await foreach (var item in source)
{
if (predicate(item))
{
yield return item;
}
}
}
public async ValueTask<T?> FirstOrDefaultAsync()
{
await foreach (var item in source)
{
return item; // Returns the very first item found
}
return default; // Returns null/default if the stream is empty
}
}
}

90
src/SharpCompress/Readers/AbstractReader.cs
View File

@@ -18,6 +18,11 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
{
private bool _completed;
private IEnumerator<TEntry>? _entriesForCurrentReadStream;
/// <summary>
/// Holds the async entry enumerator when the reader is operating in an async-only mode.
/// </summary>
private IAsyncEnumerator<TEntry>? _asyncEntriesForCurrentReadStream;
private bool _wroteCurrentEntry;
internal AbstractReader(ReaderOptions options, ArchiveType archiveType)
@@ -36,15 +41,22 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
public abstract TVolume? Volume { get; }
/// <summary>
/// Current file entry
/// Current file entry (from either sync or async enumeration).
/// </summary>
public TEntry Entry => _entriesForCurrentReadStream.NotNull().Current;
public TEntry Entry =>
_entriesForCurrentReadStream?.Current
?? _asyncEntriesForCurrentReadStream?.Current
?? throw new InvalidOperationException("No current entry is available.");
#region IDisposable Members
public virtual void Dispose()
{
_entriesForCurrentReadStream?.Dispose();
if (_asyncEntriesForCurrentReadStream is IDisposable disposable)
{
disposable.Dispose();
}
Volume?.Dispose();
}
@@ -67,6 +79,12 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
public bool MoveToNextEntry()
{
if (_asyncEntriesForCurrentReadStream is not null)
{
throw new InvalidOperationException(
$"{nameof(MoveToNextEntry)} cannot be used after {nameof(MoveToNextEntryAsync)} has been used."
);
}
if (_completed)
{
return false;
@@ -102,16 +120,17 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
{
throw new ReaderCancelledException("Reader has been cancelled.");
}
if (_entriesForCurrentReadStream is null)
if (_entriesForCurrentReadStream is null && _asyncEntriesForCurrentReadStream is null)
{
return LoadStreamForReading(RequestInitialStream());
return await LoadStreamForReadingAsync(RequestInitialStream(), cancellationToken)
.ConfigureAwait(false);
}
if (!_wroteCurrentEntry)
{
await SkipEntryAsync(cancellationToken).ConfigureAwait(false);
}
_wroteCurrentEntry = false;
if (NextEntryForCurrentStream())
if (await NextEntryForCurrentStreamAsync(cancellationToken).ConfigureAwait(false))
{
return true;
}
@@ -121,6 +140,12 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
protected bool LoadStreamForReading(Stream stream)
{
if (_asyncEntriesForCurrentReadStream is not null)
{
throw new InvalidOperationException(
$"{nameof(LoadStreamForReading)} cannot be used after {nameof(LoadStreamForReadingAsync)} has been used."
);
}
_entriesForCurrentReadStream?.Dispose();
if (stream is null || !stream.CanRead)
{
@@ -134,14 +159,69 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
return _entriesForCurrentReadStream.MoveNext();
}
/// <summary>
/// Loads the stream for reading entries asynchronously, using an async entry enumerator when available.
/// </summary>
protected async Task<bool> LoadStreamForReadingAsync(
Stream stream,
CancellationToken cancellationToken = default
)
{
// Always reset the previous async enumerator so that a new stream can be loaded cleanly.
if (_asyncEntriesForCurrentReadStream is IDisposable disposable)
{
disposable.Dispose();
}
_asyncEntriesForCurrentReadStream = null;
if (stream is null || !stream.CanRead)
{
throw new MultipartStreamRequiredException(
"File is split into multiple archives: '"
+ Entry.Key
+ "'. A new readable stream is required. Use Cancel if it was intended."
);
}
var entriesAsync = GetEntriesAsync(stream);
if (entriesAsync is null)
{
_entriesForCurrentReadStream = GetEntries(stream).GetEnumerator();
return _entriesForCurrentReadStream.MoveNext();
}
_asyncEntriesForCurrentReadStream = entriesAsync.GetAsyncEnumerator(cancellationToken);
return await _asyncEntriesForCurrentReadStream.MoveNextAsync().ConfigureAwait(false);
}
protected virtual Stream RequestInitialStream() =>
Volume.NotNull("Volume isn't loaded.").Stream;
internal virtual bool NextEntryForCurrentStream() =>
_entriesForCurrentReadStream.NotNull().MoveNext();
/// <summary>
/// Moves the current async enumerator to the next entry.
/// </summary>
internal virtual ValueTask<bool> NextEntryForCurrentStreamAsync(
CancellationToken cancellationToken = default
)
{
if (_asyncEntriesForCurrentReadStream is not null)
{
return _asyncEntriesForCurrentReadStream.MoveNextAsync();
}
return new ValueTask<bool>(NextEntryForCurrentStream());
}
protected abstract IEnumerable<TEntry> GetEntries(Stream stream);
/// <summary>
/// Optionally returns an async entry sequence for formats that support true async header parsing.
/// </summary>
protected virtual IAsyncEnumerable<TEntry>? GetEntriesAsync(Stream stream) => null;
#region Entry Skip/Write
private void SkipEntry()

101
src/SharpCompress/Readers/Zip/ZipReader.cs
View File

@@ -1,5 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
@@ -91,4 +94,102 @@ public class ZipReader : AbstractReader<ZipEntry, ZipVolume>
}
}
}
/// <summary>
/// Returns entries asynchronously for streams that only support async reads.
/// </summary>
protected override IAsyncEnumerable<ZipEntry>? GetEntriesAsync(Stream stream) =>
new ZipEntryAsyncEnumerable(_headerFactory, stream);
/// <summary>
/// Adapts an async header sequence into an async entry sequence.
/// </summary>
private sealed class ZipEntryAsyncEnumerable : IAsyncEnumerable<ZipEntry>
{
private readonly StreamingZipHeaderFactory _headerFactory;
private readonly Stream _stream;
public ZipEntryAsyncEnumerable(StreamingZipHeaderFactory headerFactory, Stream stream)
{
_headerFactory = headerFactory;
_stream = stream;
}
public IAsyncEnumerator<ZipEntry> GetAsyncEnumerator(
CancellationToken cancellationToken = default
) => new ZipEntryAsyncEnumerator(_headerFactory, _stream, cancellationToken);
}
/// <summary>
/// Yields entries from streaming ZIP headers without requiring synchronous stream reads.
/// </summary>
private sealed class ZipEntryAsyncEnumerator : IAsyncEnumerator<ZipEntry>, IDisposable
{
private readonly Stream _stream;
private readonly IAsyncEnumerator<ZipHeader> _headerEnumerator;
private ZipEntry? _current;
public ZipEntryAsyncEnumerator(
StreamingZipHeaderFactory headerFactory,
Stream stream,
CancellationToken cancellationToken
)
{
_stream = stream;
_headerEnumerator = headerFactory
.ReadStreamHeaderAsync(stream)
.GetAsyncEnumerator(cancellationToken);
}
public ZipEntry Current =>
_current ?? throw new InvalidOperationException("No current entry is available.");
/// <summary>
/// Advances to the next non-directory entry-relevant header and materializes a <see cref="ZipEntry"/>.
/// </summary>
public async ValueTask<bool> MoveNextAsync()
{
while (await _headerEnumerator.MoveNextAsync().ConfigureAwait(false))
{
var header = _headerEnumerator.Current;
switch (header.ZipHeaderType)
{
case ZipHeaderType.LocalEntry:
_current = new ZipEntry(
new StreamingZipFilePart((LocalEntryHeader)header, _stream)
);
return true;
case ZipHeaderType.DirectoryEntry:
// DirectoryEntry headers are intentionally skipped in streaming mode.
break;
case ZipHeaderType.DirectoryEnd:
_current = null;
return false;
}
}
_current = null;
return false;
}
/// <summary>
/// Disposes the underlying header enumerator.
/// </summary>
public ValueTask DisposeAsync()
{
Dispose();
return default;
}
/// <summary>
/// Disposes the underlying header enumerator.
/// </summary>
public void Dispose()
{
if (_headerEnumerator is IDisposable disposable)
{
disposable.Dispose();
}
}
}
}