From ea7d97fcee9878d3449d16d3088786cecd749bac Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 5 Aug 2026 16:20:08 +0100 Subject: [PATCH] better zip streaming info --- docs/API.md | 4 +- .../ArchiveFactory.Information.Async.cs | 46 ++++++++++++- .../Archives/ArchiveFactory.Information.cs | 67 ++++++++++++------- .../Archives/ZipArchiveInformation.cs | 9 +-- .../Common/Zip/SeekableZipFilePart.Async.cs | 5 +- .../Common/Zip/SeekableZipFilePart.cs | 13 +++- .../Zip/SeekableZipHeaderFactory.Async.cs | 32 ++++++--- .../Common/Zip/SeekableZipHeaderFactory.cs | 26 ++++++- .../Common/Zip/ZipHeaderFactory.Async.cs | 10 ++- .../Common/Zip/ZipHeaderFactory.cs | 10 ++- .../SharpCompress.Test/ArchiveFactoryTests.cs | 29 ++++++-- 11 files changed, 192 insertions(+), 59 deletions(-) diff --git a/docs/API.md b/docs/API.md index 1536c9c6..29ecb2a2 100644 --- a/docs/API.md +++ b/docs/API.md @@ -66,7 +66,7 @@ if (information is not null) { Console.WriteLine($"Entries: {information.EntryCount}"); Console.WriteLine($"Solid streams: {information.SolidStreamCount}"); - Console.WriteLine($"ZIP data descriptor entries: {information.Zip?.DataDescriptorEntryCount}"); + Console.WriteLine($"Some ZIP entry sizes arrive after their data: {information.Zip?.HasEntriesWithDeferredSizes}"); } var asyncInformation = await ArchiveFactory.InspectArchiveAsync("archive.zip", cancellationToken); @@ -96,7 +96,7 @@ The stream overloads of `DetectArchive` and `InspectArchive` preserve the suppli | `GetArchiveInformationAsync(...)` | `DetectArchiveAsync(...)` or `InspectArchiveAsync(...)` | | `Type` | `Detection.ContainerType` | | `SupportsRandomAccess` | `Detection.SupportedApis.HasFlag(ArchiveAccessMode.Archive)` | -| `ZipDataDescriptorEntryCount` | `Zip.DataDescriptorEntryCount` | +| `ZipDataDescriptorEntryCount` | `Zip.HasEntriesWithDeferredSizes` | | `SolidStreamCount` | `SolidStreamCount` | ### Creating Archives diff --git a/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs b/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs index 9c8076ef..c4fd4fa2 100644 --- a/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs +++ b/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using SharpCompress.Archives.Rar; using SharpCompress.Archives.SevenZip; +using SharpCompress.Archives.Zip; using SharpCompress.Common; using SharpCompress.Common.Ace.Headers; using SharpCompress.Common.Rar; @@ -389,7 +390,12 @@ public static partial class ArchiveFactory var entryArray = entries.ToArray(); var volumeArray = volumes.ToArray(); - var zip = GetZipInformation(entryArray); + var (zipInformation, deferredSizeEntryCount) = await GetZipInformationAsync( + archive.Type == ArchiveType.Zip, + entryArray, + cancellationToken + ) + .ConfigureAwait(false); var (isSolid, solidStreamCount) = await GetSolidInformationAsync(archive, entryArray) .ConfigureAwait(false); var isComplete = await archive.IsCompleteAsync().ConfigureAwait(false); @@ -411,7 +417,7 @@ public static partial class ArchiveFactory limitations, GetFormatVersion(volumeArray), entryArray.LongLength, - zip?.DataDescriptorEntryCount ?? 0, + deferredSizeEntryCount, physicalSize, isComplete ? GetCompressedPayloadSize(archive, entryArray) : null, isComplete && archive.Type != ArchiveType.GZip @@ -426,7 +432,7 @@ public static partial class ArchiveFactory volumeArray.Length, isComplete, GetArchiveComment(volumeArray), - zip + zipInformation ); } @@ -522,6 +528,40 @@ public static partial class ArchiveFactory _ => entries.Aggregate(0L, (total, entry) => total + entry.CompressedSize), }; + private static async ValueTask<( + ZipArchiveInformation? Information, + long DeferredSizeEntryCount + )> GetZipInformationAsync( + bool isZipArchive, + IEnumerable entries, + CancellationToken cancellationToken + ) + { + if (!isZipArchive) + { + return (null, 0); + } + + long deferredSizeEntryCount = 0; + foreach (var entry in entries.OfType()) + { + cancellationToken.ThrowIfCancellationRequested(); + var filePart = entry.Parts.OfType().Single(); + if (!filePart.HasDeferredSizes) + { + continue; + } + + var localHeader = await filePart.GetRawLocalHeaderAsync().ConfigureAwait(false); + if (localHeader.CompressedSize == 0 && localHeader.UncompressedSize == 0) + { + deferredSizeEntryCount++; + } + } + + return (new ZipArchiveInformation(deferredSizeEntryCount > 0), deferredSizeEntryCount); + } + private static async ValueTask RedetectArchiveAsync( Stream stream, long startPosition, diff --git a/src/SharpCompress/Archives/ArchiveFactory.Information.cs b/src/SharpCompress/Archives/ArchiveFactory.Information.cs index 1179a41a..264e55c9 100644 --- a/src/SharpCompress/Archives/ArchiveFactory.Information.cs +++ b/src/SharpCompress/Archives/ArchiveFactory.Information.cs @@ -258,7 +258,10 @@ public static partial class ArchiveFactory { var entries = archive.Entries.ToArray(); var volumes = archive.Volumes.ToArray(); - var zip = GetZipInformation(entries); + var (zipInformation, deferredSizeEntryCount) = GetZipInformation( + archive.Type == ArchiveType.Zip, + entries + ); var (isSolid, solidStreamCount) = GetSolidInformation(archive, entries); var isMultiVolume = GetIsMultiVolume(archive.Type, volumes); var comment = GetArchiveComment(volumes); @@ -281,7 +284,7 @@ public static partial class ArchiveFactory limitations, GetFormatVersion(volumes), entries.LongLength, - zip?.DataDescriptorEntryCount ?? 0, + deferredSizeEntryCount, physicalSize, isComplete ? GetCompressedPayloadSize(archive, entries) : null, isComplete && archive.Type != ArchiveType.GZip @@ -296,7 +299,7 @@ public static partial class ArchiveFactory volumes.Length, isComplete, comment, - zip + zipInformation ); } @@ -384,23 +387,33 @@ public static partial class ArchiveFactory ? ArchiveInformationStatus.Complete : ArchiveInformationStatus.Partial; - private static ZipArchiveInformation? GetZipInformation(IEnumerable entries) + private static ( + ZipArchiveInformation? Information, + long DeferredSizeEntryCount + ) GetZipInformation(bool isZipArchive, IEnumerable entries) { - var dataDescriptorEntryCount = entries - .OfType() - .LongCount(entry => - entry - .Parts.OfType() - .Any(part => - FlagUtility.HasFlag( - part.Header.Flags, - SharpCompress.Common.Zip.Headers.HeaderFlags.UsePostDataDescriptor - ) - ) - ); - return dataDescriptorEntryCount == 0 && !entries.OfType().Any() - ? null - : new ZipArchiveInformation(dataDescriptorEntryCount); + if (!isZipArchive) + { + return (null, 0); + } + + long deferredSizeEntryCount = 0; + foreach (var entry in entries.OfType()) + { + var filePart = entry.Parts.OfType().Single(); + if (!filePart.HasDeferredSizes) + { + continue; + } + + var localHeader = filePart.GetRawLocalHeader(); + if (localHeader.CompressedSize == 0 && localHeader.UncompressedSize == 0) + { + deferredSizeEntryCount++; + } + } + + return (new ZipArchiveInformation(deferredSizeEntryCount > 0), deferredSizeEntryCount); } private static (bool IsSolid, long SolidStreamCount) GetSolidInformation( @@ -563,7 +576,7 @@ public static partial class ArchiveFactory private readonly ArchiveDetection detection; private long? compressedPayloadSize = 0; private long? uncompressedPayloadSize = 0; - private long dataDescriptorEntryCount; + private long deferredSizeEntryCount; private long entriesWithUnknownSizeCount; private bool isEncrypted; private ArchiveInformationLimitations limitations; @@ -595,15 +608,15 @@ public static partial class ArchiveFactory EntryCount++; isEncrypted |= entry.IsEncrypted; - var usesDataDescriptor = UsesZipDataDescriptor(entry); - if (usesDataDescriptor) + var hasDeferredSizes = HasDeferredSizes(entry); + if (hasDeferredSizes) { - dataDescriptorEntryCount++; + deferredSizeEntryCount++; } if ( detection.ContainerType == ArchiveType.Lzw - || usesDataDescriptor + || hasDeferredSizes || !TryGetSize(entry, out var size) ) { @@ -650,7 +663,7 @@ public static partial class ArchiveFactory true, null, detection.ContainerType == ArchiveType.Zip - ? new ZipArchiveInformation(dataDescriptorEntryCount) + ? new ZipArchiveInformation(deferredSizeEntryCount > 0) : null ); @@ -668,7 +681,7 @@ public static partial class ArchiveFactory } } - private static bool UsesZipDataDescriptor(IEntry entry) => + private static bool HasDeferredSizes(IEntry entry) => entry is ZipEntry zipEntry && zipEntry .Parts.OfType() @@ -677,6 +690,8 @@ public static partial class ArchiveFactory part.Header.Flags, SharpCompress.Common.Zip.Headers.HeaderFlags.UsePostDataDescriptor ) + && part.Header.CompressedSize == 0 + && part.Header.UncompressedSize == 0 ); } } diff --git a/src/SharpCompress/Archives/ZipArchiveInformation.cs b/src/SharpCompress/Archives/ZipArchiveInformation.cs index c7160903..80bc76cd 100644 --- a/src/SharpCompress/Archives/ZipArchiveInformation.cs +++ b/src/SharpCompress/Archives/ZipArchiveInformation.cs @@ -5,11 +5,12 @@ namespace SharpCompress.Archives; /// public sealed class ZipArchiveInformation { - internal ZipArchiveInformation(long dataDescriptorEntryCount) => - DataDescriptorEntryCount = dataDescriptorEntryCount; + internal ZipArchiveInformation(bool hasEntriesWithDeferredSizes) => + HasEntriesWithDeferredSizes = hasEntriesWithDeferredSizes; /// - /// Gets the number of entries whose local header defers its CRC and sizes to a data descriptor. + /// Gets whether any entry sizes are available only after reading the entry data. + /// When , a forward-only reader can initially report zero sizes for those entries. /// - public long DataDescriptorEntryCount { get; } + public bool HasEntriesWithDeferredSizes { get; } } diff --git a/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs b/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs index da9ea359..f4ff6799 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs @@ -23,6 +23,9 @@ internal partial class SeekableZipFilePart [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask LoadLocalHeaderAsync(CancellationToken cancellationToken = default) => Header = await _headerFactory - .GetLocalHeaderAsync(BaseStream, (DirectoryEntryHeader)Header) + .GetLocalHeaderAsync(BaseStream, _directoryEntryHeader) .ConfigureAwait(false); + + internal ValueTask GetRawLocalHeaderAsync() => + _headerFactory.GetRawLocalHeaderAsync(BaseStream, _directoryEntryHeader); } diff --git a/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs b/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs index bed94da3..166c55b8 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs @@ -8,6 +8,7 @@ internal partial class SeekableZipFilePart : ZipFilePart { private bool _isLocalHeaderLoaded; private readonly SeekableZipHeaderFactory _headerFactory; + private readonly DirectoryEntryHeader _directoryEntryHeader; internal SeekableZipFilePart( SeekableZipHeaderFactory headerFactory, @@ -15,7 +16,17 @@ internal partial class SeekableZipFilePart : ZipFilePart Stream stream, CompressionProviderRegistry compressionProviders ) - : base(header, stream, compressionProviders) => _headerFactory = headerFactory; + : base(header, stream, compressionProviders) + { + _headerFactory = headerFactory; + _directoryEntryHeader = header; + } + + internal LocalEntryHeader GetRawLocalHeader() => + _headerFactory.GetRawLocalHeader(BaseStream, _directoryEntryHeader); + + internal bool HasDeferredSizes => + FlagUtility.HasFlag(_directoryEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor); protected override Stream CreateBaseStream() { diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs index 11e7eec4..e9065f80 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs @@ -146,17 +146,27 @@ internal sealed partial class SeekableZipHeaderFactory throw new ArchiveOperationException(); } - // populate fields only known from the DirectoryEntryHeader - localEntryHeader.HasData = directoryEntryHeader.HasData; - localEntryHeader.ExternalFileAttributes = directoryEntryHeader.ExternalFileAttributes; - localEntryHeader.Comment = directoryEntryHeader.Comment; - - if (FlagUtility.HasFlag(localEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor)) - { - localEntryHeader.Crc = directoryEntryHeader.Crc; - localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; - localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; - } + PopulateDirectoryEntryMetadata(localEntryHeader, directoryEntryHeader); return localEntryHeader; } + + internal async ValueTask GetRawLocalHeaderAsync( + Stream stream, + DirectoryEntryHeader directoryEntryHeader + ) + { + stream.Seek(directoryEntryHeader.RelativeOffsetOfEntryHeader, SeekOrigin.Begin); +#if NET8_0_OR_GREATER + await using var reader = new AsyncBinaryReader(stream, leaveOpen: true); +#else + using var reader = new AsyncBinaryReader(stream, leaveOpen: true); +#endif + var signature = await reader.ReadUInt32Async().ConfigureAwait(false); + if (signature != ENTRY_HEADER_BYTES) + { + throw new ArchiveOperationException(); + } + + return await ReadLocalHeader(reader).ConfigureAwait(false); + } } diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs index 7a88a480..c20163b9 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs @@ -151,6 +151,31 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory throw new ArchiveOperationException(); } + PopulateDirectoryEntryMetadata(localEntryHeader, directoryEntryHeader); + return localEntryHeader; + } + + internal LocalEntryHeader GetRawLocalHeader( + Stream stream, + DirectoryEntryHeader directoryEntryHeader + ) + { + stream.Seek(directoryEntryHeader.RelativeOffsetOfEntryHeader, SeekOrigin.Begin); + var reader = new BinaryReader(stream); + var signature = reader.ReadUInt32(); + if (signature != ENTRY_HEADER_BYTES) + { + throw new ArchiveOperationException(); + } + + return ReadLocalHeader(reader); + } + + private static void PopulateDirectoryEntryMetadata( + LocalEntryHeader localEntryHeader, + DirectoryEntryHeader directoryEntryHeader + ) + { // populate fields only known from the DirectoryEntryHeader localEntryHeader.HasData = directoryEntryHeader.HasData; localEntryHeader.ExternalFileAttributes = directoryEntryHeader.ExternalFileAttributes; @@ -163,6 +188,5 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; } - return localEntryHeader; } } diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs index bc5944b4..bb420a59 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs @@ -20,8 +20,7 @@ internal partial class ZipHeaderFactory { case ENTRY_HEADER_BYTES: { - var entryHeader = new LocalEntryHeader(_archiveEncoding); - await entryHeader.Read(reader).ConfigureAwait(false); + var entryHeader = await ReadLocalHeader(reader).ConfigureAwait(false); await LoadHeaderAsync(entryHeader, reader.BaseStream).ConfigureAwait(false); _lastEntryHeader = entryHeader; @@ -87,6 +86,13 @@ internal partial class ZipHeaderFactory } } + protected async ValueTask ReadLocalHeader(AsyncBinaryReader reader) + { + var entryHeader = new LocalEntryHeader(_archiveEncoding); + await entryHeader.Read(reader).ConfigureAwait(false); + return entryHeader; + } + /// /// Loads encryption metadata and stream positioning for a header using async reads where needed. /// diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs index e1ebb8c1..f98620f9 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs @@ -42,8 +42,7 @@ internal partial class ZipHeaderFactory { case ENTRY_HEADER_BYTES: { - var entryHeader = new LocalEntryHeader(_archiveEncoding); - entryHeader.Read(reader); + var entryHeader = ReadLocalHeader(reader); LoadHeader(entryHeader, reader.BaseStream); _lastEntryHeader = entryHeader; @@ -109,6 +108,13 @@ internal partial class ZipHeaderFactory } } + protected LocalEntryHeader ReadLocalHeader(BinaryReader reader) + { + var entryHeader = new LocalEntryHeader(_archiveEncoding); + entryHeader.Read(reader); + return entryHeader; + } + internal static bool IsHeader(uint headerBytes) { switch (headerBytes) diff --git a/tests/SharpCompress.Test/ArchiveFactoryTests.cs b/tests/SharpCompress.Test/ArchiveFactoryTests.cs index 5ea9d3c0..2de784f9 100644 --- a/tests/SharpCompress.Test/ArchiveFactoryTests.cs +++ b/tests/SharpCompress.Test/ArchiveFactoryTests.cs @@ -760,11 +760,13 @@ public class ArchiveFactoryTests : TestBase } [Theory] - [InlineData("Zip.none.datadescriptors.zip", 2)] - [InlineData("Zip.deflate.zip", 0)] - public void InspectArchive_ZipDataDescriptorEntryCount( + [InlineData("Zip.none.datadescriptors.zip", true, 2)] + [InlineData("Zip.deflate.dd.zip", true, 3)] + [InlineData("Zip.deflate.zip", false, 0)] + public void InspectArchive_ZipReportsEntriesWithDeferredSizes( string archiveName, - long expectedDataDescriptorEntryCount + bool expectedHasEntriesWithDeferredSizes, + long expectedUnknownSizeEntryCount ) { using var stream = File.OpenRead(GetTestArchivePath(archiveName)); @@ -774,8 +776,23 @@ public class ArchiveFactoryTests : TestBase Assert.NotNull(info); Assert.Equal(ArchiveType.Zip, info.Detection.ContainerType); Assert.NotNull(info.Zip); - Assert.Equal(expectedDataDescriptorEntryCount, info.Zip.DataDescriptorEntryCount); - Assert.Equal(expectedDataDescriptorEntryCount, info.EntriesWithUnknownSizeCount); + Assert.Equal(expectedHasEntriesWithDeferredSizes, info.Zip.HasEntriesWithDeferredSizes); + Assert.Equal(expectedUnknownSizeEntryCount, info.EntriesWithUnknownSizeCount); + } + + [Theory] + [InlineData("Zip.none.datadescriptors.zip", true)] + [InlineData("Zip.deflate.zip", false)] + public async ValueTask InspectArchiveAsync_ZipReportsEntriesWithDeferredSizes( + string archiveName, + bool expectedHasEntriesWithDeferredSizes + ) + { + var info = await ArchiveFactory.InspectArchiveAsync(GetTestArchivePath(archiveName)); + + Assert.NotNull(info); + Assert.NotNull(info.Zip); + Assert.Equal(expectedHasEntriesWithDeferredSizes, info.Zip.HasEntriesWithDeferredSizes); } [Theory]