diff --git a/Directory.Packages.props b/Directory.Packages.props index aa5098f8..572972e1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,24 +1,24 @@ - - + + - + - + - + - + diff --git a/build/packages.lock.json b/build/packages.lock.json index 303dd24e..5933ae13 100644 --- a/build/packages.lock.json +++ b/build/packages.lock.json @@ -4,9 +4,9 @@ "net10.0": { "Bullseye": { "type": "Direct", - "requested": "[6.1.0, )", - "resolved": "6.1.0", - "contentHash": "fltnAJDe0BEX5eymXGUq+il2rSUA0pHqUonNDRH2TrvRu8SkU17mYG0IVpdmG2ibtfhdjNrv4CuTCxHOwcozCA==" + "requested": "[6.2.0, )", + "resolved": "6.2.0", + "contentHash": "YWQxe+pvudGSzW1QqnRUlr9XV0xtlNvMxorVaFPqoMH8qAYNxniMQKGrPSiQkgEy0ckn/TAq98E+DIwTSR7vbQ==" }, "Glob": { "type": "Direct", @@ -25,13 +25,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -48,22 +48,22 @@ }, "SimpleExec": { "type": "Direct", - "requested": "[13.0.0, )", - "resolved": "13.0.0", - "contentHash": "zcCR1pupa1wI1VqBULRiQKeHKKZOuJhi/K+4V5oO+rHJZlaOD53ViFo1c3PavDoMAfSn/FAXGAWpPoF57rwhYg==" + "requested": "[13.1.0, )", + "resolved": "13.1.0", + "contentHash": "Q/xUWjrB98Aq+guCJtGFAHmnH00J3Ux7Q/gbnVkuh1U4FHZ2/3PgBHTcZuuhs2cSv5pNgZeHEmGwLgSjjbFlrw==" }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -73,13 +73,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" } } } diff --git a/docs/API.md b/docs/API.md index 80143e83..7da04787 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,7 +10,7 @@ Quick reference for commonly used SharpCompress APIs. // Auto-detect format using (var reader = ReaderFactory.OpenReader(stream)) { - // Works with Zip, Tar, GZip, Rar, 7Zip, etc. + // Works with Zip, Tar, GZip, Rar, etc. Reading 7Zip needs the Archive API } // Specific format - Archive API @@ -50,18 +50,26 @@ if (ArchiveFactory.IsArchive("archive.zip", out var archiveType)) Console.WriteLine($"Detected {archiveType}"); } -// Detect capabilities before choosing Archive API vs Reader API -var info = ArchiveFactory.GetArchiveInformation("archive.arc"); -if (info is not null) +// Detect the format and choose an API without enumerating entries. +var detection = ArchiveFactory.DetectArchive("archive.tar.xz"); +if (detection is not null) { - Console.WriteLine($"Type: {info.Type}"); - Console.WriteLine($"Supports random access: {info.SupportsRandomAccess}"); + Console.WriteLine($"Container: {detection.ContainerType}"); + Console.WriteLine($"Outer compression: {detection.OuterCompressionType}"); + Console.WriteLine($"Supported APIs: {detection.SupportedApis}"); } -var asyncInfo = await ArchiveFactory.GetArchiveInformationAsync( - "archive.zip", - cancellationToken -); +// InspectArchive parses complete archive metadata. It can be expensive for +// sequential formats such as TAR and compressed TAR. +var information = ArchiveFactory.InspectArchive("archive.zip"); +if (information is not null) +{ + Console.WriteLine($"Entries: {information.EntryCount}"); + Console.WriteLine($"Solid streams: {information.SolidStreamCount}"); + Console.WriteLine($"Some ZIP entry sizes arrive after their data: {information.Zip?.HasEntriesWithDeferredSizes}"); +} + +var asyncInformation = await ArchiveFactory.InspectArchiveAsync("archive.zip", cancellationToken); // Multi-volume archives var parts = ArchiveFactory.GetFileParts("archive.part1.rar") @@ -73,7 +81,25 @@ using (var archive = ArchiveFactory.OpenArchive(parts)) } ``` -`ArchiveInformation.SupportsRandomAccess` is `true` when the detected format supports `IArchive` random access. It is `false` for reader-only formats such as Ace, Arc, Arj, and standalone LZW, where `ReaderFactory.OpenReader` should be used instead. Compressed tar wrappers such as `.tar.gz` and `.tar.xz` are also reader-only; `ArchiveFactory.GetArchiveInformation` returns `null` for them and `ArchiveFactory.OpenArchive` does not open them as the outer compression wrapper. Use `ReaderFactory.OpenReader` or `TarReader.OpenReader` for those files. +`DetectArchive` identifies the logical container, its outer compression wrapper, and whether the Archive and Reader APIs are available without enumerating entries. Compressed TAR wrappers such as `.tar.gz` and `.tar.xz` are identified as `ArchiveType.Tar` with an outer compression type and `ArchiveAccessMode.Reader`. + +`InspectArchive` enumerates metadata and returns `ArchiveInformation`. It reports `Partial` status for missing volumes or encrypted headers without a password; malformed archives and incorrect passwords throw. ZIP-specific metadata is exposed through `ArchiveInformation.Zip`. + +Detection and inspection result types are in the `SharpCompress.Detection` namespace. + +The stream overloads of `DetectArchive` and `InspectArchive` preserve the supplied stream's position and leave it open, including when `ReaderOptions.LeaveStreamOpen` is `false`. + +#### Migrating from `GetArchiveInformation` + +| Removed API | Replacement | +| --- | --- | +| `GetArchiveInformation(...)` used only to identify a format | `DetectArchive(...)` | +| `GetArchiveInformation(...)` used to enumerate archive details | `InspectArchive(...)` | +| `GetArchiveInformationAsync(...)` | `DetectArchiveAsync(...)` or `InspectArchiveAsync(...)` | +| `Type` | `Detection.ContainerType` | +| `SupportsRandomAccess` | `Detection.SupportedApis.HasFlag(ArchiveAccessMode.Archive)` | +| `ZipDataDescriptorEntryCount` | `Zip.HasEntriesWithDeferredSizes` | +| `SolidStreamCount` | `SolidStreamCount` | ### Creating Archives @@ -230,7 +256,7 @@ using (var reader = ReaderFactory.OpenReader(stream)) { while (reader.MoveToNextEntry()) { - IArchiveEntry entry = reader.Entry; + IEntry entry = reader.Entry; if (!entry.IsDirectory) { @@ -332,6 +358,10 @@ var hinted = ReaderOptions.ForExternalStream.WithExtensionHint("tar.gz"); // Increase for non-seekable streams with large detection probes, such as SFX RAR var buffered = ReaderOptions.ForExternalStream.WithRewindableBufferSize(1_048_576); +// Opt in to formats' optional parallel decode (e.g. 7-Zip's automatic parallel LZMA2 +// solid-folder decode); default is sequential decoding +var parallel = ReaderOptions.ForExternalStream.WithEnableParallelism(true); + // Extraction presets var safeOptions = ExtractionOptions.SafeExtract; // No overwrite var flatOptions = ExtractionOptions.FlatExtract; // No directory structure @@ -362,6 +392,7 @@ var options = new ReaderOptions DisableCheckIncomplete = false, BufferSize = 81920, RewindableBufferSize = 1_048_576, + EnableParallelism = false, }; var extractionOptions = new ExtractionOptions @@ -444,6 +475,8 @@ var options = new WriterOptions(CompressionType.Deflate) archive.SaveTo("output.zip", options); ``` +`WriterOptions.EnableParallelism` (also available on `ZipWriterOptions`, `TarWriterOptions`, `GZipWriterOptions`, and `SevenZipWriterOptions`) opts in to a format's optional parallel encode. No writer currently implements parallel encoding, so this is a no-op today; it is reserved for future use and mirrors `ReaderOptions.EnableParallelism`. + ### Extraction behavior ```csharp @@ -463,6 +496,8 @@ using (var archive = ZipArchive.OpenArchive("file.zip")) `CheckCrc` validates archive-level payload checksums when the format stores reliable metadata, such as ZIP CRC32 values. Formats without payload checksums skip this validation. Decompressor integrity checks that are required to decode a stream may still fail even when `CheckCrc` is disabled. +When using `SymbolicLinkHandler`, directory extraction rejects link targets outside the extraction root and never follows symbolic links or reparse points while extracting later entries. The handler itself remains trusted application code. + ### Options matrix ```text @@ -576,7 +611,6 @@ using (var writer = WriterFactory.OpenWriter(stream, ArchiveType.Tar, Compressio ArchiveType.Zip ArchiveType.Tar ArchiveType.GZip -ArchiveType.BZip2 ArchiveType.Rar ArchiveType.SevenZip ArchiveType.Arc @@ -600,11 +634,11 @@ try archive.WriteToDirectory(@"C:\output"); } } -catch (PasswordRequiredException) +catch (CryptographicException) { Console.WriteLine("Password required"); } -catch (InvalidArchiveException) +catch (ArchiveException) { Console.WriteLine("Archive is invalid"); } diff --git a/docs/FORMATS.md b/docs/FORMATS.md index 22a7080a..3a2cde50 100644 --- a/docs/FORMATS.md +++ b/docs/FORMATS.md @@ -31,7 +31,7 @@ 4. The 7Zip format doesn't allow for reading as a forward-only stream, so 7Zip read support is only through the Archive API. Writing is supported through SevenZipWriter for non-solid archives with LZMA/LZMA2 and requires a seekable output stream. See [7Zip Format Notes](#7zip-format-notes) for details on async extraction behavior. 5. LZip has no support for extra data like the file name or timestamp. There is a default filename used when looking at the entry Key on the archive. -`ArchiveFactory.GetArchiveInformation(...).SupportsRandomAccess` is `true` when the detected format has an Archive API in this table. It is `false` for reader-only formats such as Ace, Arc, Arj, and standalone LZW. Compressed tar wrappers are supported by `ReaderFactory`/`TarReader`, not by `ArchiveFactory`/`TarArchive`; ArchiveFactory detection blocks them instead of opening the outer compression wrapper as a standalone archive. +`ArchiveFactory.DetectArchive(...)` reports which APIs in this table are available through `SharpCompress.Detection.ArchiveDetection.SupportedApis`. Reader-only formats include Ace, Arc, Arj, and standalone LZW. Compressed TAR wrappers are detected as a TAR container with an outer compression type and support the Reader API, not the Archive API. Use `ArchiveFactory.InspectArchive(...)` when complete archive metadata is required. ### Zip Format Notes diff --git a/docs/SYNC_METHOD_GENERATION.md b/docs/SYNC_METHOD_GENERATION.md index aac905ef..9bf9ca1b 100644 --- a/docs/SYNC_METHOD_GENERATION.md +++ b/docs/SYNC_METHOD_GENERATION.md @@ -25,7 +25,13 @@ the verification workflow, and the record of what is blocked and why. | 0 | `Archives/IArchiveEntryExtensions` (5 methods) | done — `c303856c` | | 1 | `Compressors/Filters/Filter` + 6 XZ branch filters + `Lzma2Filter` (9 methods, 147 lines) | done — verified identical on net48 + net10.0 | | 2 | `IO/` leaf stream shims + `Common/EntryStream` (17 methods, 190 lines) | done — verified identical on net48 + net10.0 | -| 3–12 | below | to do | +| 3–5 | below | to do | +| 6 | ZIP parts & header factories | 6a done; header factories remain | +| 7 | Concrete writers | ZIP done; Tar, SevenZip, and remaining writers to do | +| 8 | 7-Zip | to do | +| 9 | Streaming compressors | Deflate, Zlib, and Deflate64 done; remaining compressors to do | +| 10–11 | below | to do | +| 12 | Providers | registry done; provider implementations remain | | Rar reader unification | below | to do, needs its own design review | --- @@ -207,16 +213,17 @@ the async one `new byte[4]`, and the method is 4 lines — `SYNC_ONLY` would be `Arj/HuffmanTree`, `Arj/BitReader`, `Arj/LhaStream` (11 pairs, all ≥0.99), `Arj/LHDecoderStream`, `Squeezed/BitReader`. Best win per line of review — pure parsing, all clean. -### 6 — Zip parts & header factories · ~250 lines · med risk +### 6 — Zip parts & header factories · ~250 lines · med risk · **in progress** 6a: `ZipFilePart` (incl. `GetCryptoStreamAsync`), `SeekableZipFilePart`, `StreamingZipFilePart`, -`GZipFilePart`, `Zip/Headers/ZipFileEntry`, `PkwareTraditionalCryptoStream`, `WinzipAesCryptoStream`. +`Zip/Headers/ZipFileEntry`, `PkwareTraditionalCryptoStream`, and `WinzipAesCryptoStream` — **done**. +`GZipFilePart` was completed in the prior GZip batch. 6b: `ZipHeaderFactory` (`LoadHeaderAsync` identical), `SeekableZipHeaderFactory`. -### 7 — Concrete writers · ~250 lines · med risk +### 7 — Concrete writers · ~250 lines · med risk · **in progress** -`ZipWriter`, `ZipWritingStream` (`GetWriteStreamAsync` is a 131-line duplicate), `TarWriter`, -`SevenZipWriter`, `GZipWriter`. +`ZipWriter` and `ZipWritingStream` (`GetWriteStreamAsync` is a 131-line duplicate) — **done**. +`TarWriter` and `SevenZipWriter` remain. `GZipWriter` was completed in the prior GZip batch. This is the **reachable Writers win**: `AbstractWriter` already implements *both* `IWriter` and `IAsyncWriter` (`AbstractWriter.cs:11`), so `ZipWriter.Async.cs:52 @@ -233,14 +240,15 @@ attempt `IWriterExtensions` (see Blocked). Clean: these use the sync `DataReader`, not an async-only reader type. Keep the file split (`ArchiveReader.cs` is 1,377 lines). -### 9 — Streaming compressors · ~300 lines · med-high risk +### 9 — Streaming compressors · ~300 lines · med-high risk · **in progress** -`Deflate64Stream`, `Deflate/ZlibBaseStream` (`ReadAsync` 192 lines, `WriteAsync`, `FlushAsync`), -`DeflateStream`, `GZipStream`, `ZlibStream`, `ZStandard/*`, `Reduce`, `Explode`, `Lzw/LzwStream` -(`ReadAsync` 208 lines), `RLE90`, `Shrink`, `ArcLzw`. +`Deflate64Stream`, `Deflate/ZlibBaseStream` (`ReadAsync`, `WriteAsync`, `FlushAsync`), +`DeflateStream`, and `ZlibStream` — **done**. `GZipStream` was completed in the prior GZip batch. +`ZStandard/*`, `Reduce`, `Explode`, `Lzw/LzwStream` (`ReadAsync` 208 lines), `RLE90`, `Shrink`, +and `ArcLzw` remain. -- `ZlibBaseStream` needs `partial` added. -- `[SkipSyncVersion]` on all four Deflate-family `DisposeAsync`. +- `ZlibBaseStream` is partial. Deflate-family `DisposeAsync`, byte methods, unmatched memory + overloads, GZip-header parsing, and finalization remain hand-written. - `RunLength90Stream` is the canonical `SYNC_ONLY` case (see below). - `GZipCompressionProvider.cs:41` returns `new ValueTask(...)` from a non-`async` method — not a Zomp rewrite; make it `async` first. (Non-`async` methods that just *return* an `XAsync(...)` @@ -264,11 +272,12 @@ generated `ReadByte()`/`WriteByte()` are emitted without `override` and would hi Fully clean — these use raw `Stream`, **not** the async reader types. The least readable diffs in the repo, so do them last of the mechanical work. -### 12 — Providers · ~80 lines · low risk +### 12 — Providers · ~80 lines · low risk · **in progress** -`CompressionProviderRegistry` (4 identical pairs), `Default/*` providers (`XStream.CreateAsync(...)` -→ `Create(...)` maps cleanly), `CompressionProviderBase`, -`ContextRequiredDecompressionProviderBase`. Blocked on the `GZipCompressionProvider` rewrite above. +`CompressionProviderRegistry` (4 identical pairs) — **done**. `Default/*` providers +(`XStream.CreateAsync(...)` → `Create(...)` maps cleanly), `CompressionProviderBase`, and +`ContextRequiredDecompressionProviderBase` remain. The `GZipCompressionProvider` rewrite was +completed in the prior GZip batch. --- diff --git a/docs/USAGE.md b/docs/USAGE.md index 9666d43f..91e6331a 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -227,16 +227,18 @@ using (var archive = ArchiveFactory.OpenArchive("archive.zip")) } ``` -### Use ArchiveInformation to choose the right API +### Detect the format and choose the right API ```C# +using SharpCompress.Detection; + var archivePath = "archive.arc"; -var info = ArchiveFactory.GetArchiveInformation(archivePath); -if (info is null) +var detection = ArchiveFactory.DetectArchive(archivePath); +if (detection is null) { Console.WriteLine("Not a supported archive"); } -else if (info.SupportsRandomAccess) +else if (detection.SupportedApis.HasFlag(ArchiveAccessMode.Archive)) { using var archive = ArchiveFactory.OpenArchive(archivePath); archive.WriteToDirectory(@"D:\output"); @@ -248,7 +250,25 @@ else } ``` -`SupportsRandomAccess` is `false` for reader-only formats such as Ace, Arc, Arj, and standalone LZW. Use the Reader API for those formats. +`DetectArchive` does not enumerate entries. It reports both Archive and Reader API availability; reader-only formats include Ace, Arc, Arj, standalone LZW, and compressed TAR wrappers. + +### Inspect archive metadata + +```C# +var information = ArchiveFactory.InspectArchive(archivePath); +if (information is not null) +{ + Console.WriteLine($"Entries: {information.EntryCount}"); + Console.WriteLine($"Compressed bytes: {information.CompressedPayloadSize}"); + Console.WriteLine($"Solid streams: {information.SolidStreamCount}"); +} +``` + +`InspectArchive` parses archive metadata, which can require a complete sequential scan for TAR and compressed TAR files. It returns partial information for encrypted headers without a password or missing archive parts; inspect `Status` and `Limitations` before using nullable metadata values. + +Detection and inspection result types are in the `SharpCompress.Detection` namespace. + +The stream overload preserves the caller's current position and leaves the supplied stream open. ### Open multi-volume archives diff --git a/global.json b/global.json index 92a71249..e784f38f 100644 --- a/global.json +++ b/global.json @@ -3,4 +3,4 @@ "version": "10.0.302", "rollForward": "disable" } -} +} \ No newline at end of file diff --git a/src/SharpCompress/Archives/ArchiveFactory.Detection.Async.cs b/src/SharpCompress/Archives/ArchiveFactory.Detection.Async.cs new file mode 100644 index 00000000..04cc7886 --- /dev/null +++ b/src/SharpCompress/Archives/ArchiveFactory.Detection.Async.cs @@ -0,0 +1,369 @@ +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives.Tar; +using SharpCompress.Common; +using SharpCompress.Detection; +using SharpCompress.Factories; +using SharpCompress.IO; +using SharpCompress.Providers; +using SharpCompress.Readers; + +namespace SharpCompress.Archives; + +public static partial class ArchiveFactory +{ + /// + /// Identifies the archive at the given file path without enumerating its entries. + /// + /// Path to the archive file. + /// Cancellation token. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask DetectArchiveAsync( + string filePath, + CancellationToken cancellationToken = default + ) => + await DetectArchiveAsync(filePath, ReaderOptions.ForFilePath, cancellationToken) + .ConfigureAwait(false); + + /// + /// Identifies the archive at the given file path without enumerating its entries. + /// + /// Path to the archive file. + /// Options controlling archive detection. + /// Cancellation token. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask DetectArchiveAsync( + string filePath, + ReaderOptions? readerOptions, + CancellationToken cancellationToken = default + ) + { + filePath.NotNullOrEmpty(nameof(filePath)); + using Stream stream = File.OpenRead(filePath); + return await DetectArchiveAsync( + stream, + readerOptions ?? ReaderOptions.ForFilePath, + cancellationToken + ) + .ConfigureAwait(false); + } + + /// + /// Identifies the archive in the given stream without enumerating its entries. + /// + /// A readable and seekable stream positioned at the start of the archive. + /// Cancellation token. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask DetectArchiveAsync( + Stream stream, + CancellationToken cancellationToken = default + ) => + await DetectArchiveAsync(stream, ReaderOptions.ForExternalStream, cancellationToken) + .ConfigureAwait(false); + + /// + /// Identifies the archive in the given stream without enumerating its entries. + /// + /// A readable and seekable stream positioned at the start of the archive. + /// Options controlling archive detection. + /// Cancellation token. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask DetectArchiveAsync( + Stream stream, + ReaderOptions? readerOptions, + CancellationToken cancellationToken = default + ) + { + stream.RequireReadable(); + stream.RequireSeekable(); + + return await TryDetectArchiveAsync( + stream, + readerOptions ?? ReaderOptions.ForExternalStream, + cancellationToken + ) + .ConfigureAwait(false); + } + + internal static ValueTask FindFactoryAsync( + string filePath, + CancellationToken cancellationToken = default + ) + where T : IFactory + { + filePath.NotNullOrEmpty(nameof(filePath)); + return FindFactoryAsync(new FileInfo(filePath), cancellationToken); + } + + internal static async ValueTask FindFactoryAsync( + FileInfo fileInfo, + CancellationToken cancellationToken = default + ) + where T : IFactory + { + fileInfo.NotNull(nameof(fileInfo)); + using Stream stream = fileInfo.OpenRead(); + return await FindFactoryAsync(stream, cancellationToken).ConfigureAwait(false); + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask FindFactoryAsync( + FileInfo fileInfo, + ReaderOptions readerOptions, + CancellationToken cancellationToken + ) + where T : IFactory + { + fileInfo.NotNull(nameof(fileInfo)); + using Stream stream = fileInfo.OpenRead(); + return await FindFactoryAsync(stream, readerOptions, cancellationToken) + .ConfigureAwait(false); + } + + internal static async ValueTask FindFactoryAsync( + Stream stream, + CancellationToken cancellationToken = default + ) + where T : IFactory + { + stream.RequireReadable(); + stream.RequireSeekable(); + + // Use the shared async detection loop over all factories. If the matched factory + // implements T we return it; otherwise (or if nothing matched) we fall through + // to the same "unsupported format" exception that the original code produced, + // listing the T-typed factories as the hint for the caller. + return await FindFactoryAsync(stream, ReaderOptions.ForExternalStream, cancellationToken) + .ConfigureAwait(false); + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask FindFactoryAsync( + Stream stream, + ReaderOptions readerOptions, + CancellationToken cancellationToken + ) + where T : IFactory + { + stream.RequireReadable(); + stream.RequireSeekable(); + + var factory = await TryFindFactoryAsync(stream, readerOptions, cancellationToken) + .ConfigureAwait(false); + if (factory is T typedFactory) + { + return typedFactory; + } + + var extensions = string.Join(", ", Factory.Factories.OfType().Select(item => item.Name)); + + throw new ArchiveOperationException( + $"Cannot determine compressed stream type. Supported Archive Formats: {extensions}" + ); + } + + /// + /// Iterates all registered factories and returns the first one whose + /// recognises the stream, or . + /// Stream position is restored to its value at entry on both success and failure. + /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask TryFindFactoryAsync( + Stream stream, + ReaderOptions readerOptions, + CancellationToken cancellationToken + ) + { + var startPosition = stream.Position; + + try + { + foreach (var factory in Factory.Factories) + { + stream.Seek(startPosition, SeekOrigin.Begin); + var isArchive = await factory + .IsArchiveAsync(stream, readerOptions, cancellationToken) + .ConfigureAwait(false); + + if (isArchive) + { + stream.Seek(startPosition, SeekOrigin.Begin); + if ( + await IsCompressedTarAsync( + stream, + factory, + readerOptions, + cancellationToken + ) + .ConfigureAwait(false) + ) + { + continue; + } + + return factory; + } + } + + return null; + } + finally + { + stream.Seek(startPosition, SeekOrigin.Begin); + } + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask IsCompressedTarAsync( + Stream stream, + IFactory factory, + ReaderOptions readerOptions, + CancellationToken cancellationToken + ) => + GetCompressedTarType(factory) is { } compressionType + && await IsCompressedTarAsync(stream, readerOptions, compressionType, cancellationToken) + .ConfigureAwait(false); + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask IsCompressedTarAsync( + Stream stream, + ReaderOptions readerOptions, + CompressionType compressionType, + CancellationToken cancellationToken + ) + { + using var nonDisposingStream = SharpCompressStream.CreateNonDisposing(stream); + var testStream = + compressionType == CompressionType.GZip + ? await readerOptions + .Providers.CreateDecompressStreamAsync( + compressionType, + nonDisposingStream, + CompressionContext + .FromStream(nonDisposingStream) + .WithReaderOptions(readerOptions), + cancellationToken + ) + .ConfigureAwait(false) + : await readerOptions + .Providers.CreateDecompressStreamAsync( + compressionType, + nonDisposingStream, + cancellationToken + ) + .ConfigureAwait(false); + + try + { + return await TarArchive + .IsTarFileAsync(testStream, cancellationToken) + .ConfigureAwait(false); + } + finally + { + DisposeProbeStream(testStream); + } + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask TryDetectArchiveAsync( + Stream stream, + ReaderOptions readerOptions, + CancellationToken cancellationToken + ) + { + var startPosition = stream.Position; + + try + { + foreach (var factory in Factory.Factories) + { + stream.Seek(startPosition, SeekOrigin.Begin); + if ( + !await factory + .IsArchiveAsync(stream, readerOptions, cancellationToken) + .ConfigureAwait(false) + ) + { + continue; + } + + if (GetCompressedTarType(factory) is { } compressionType) + { + stream.Seek(startPosition, SeekOrigin.Begin); + if ( + await IsCompressedTarAsync( + stream, + readerOptions, + compressionType, + cancellationToken + ) + .ConfigureAwait(false) + ) + { + return CreateCompressedTarDetection(compressionType); + } + } + + return CreateDetection(factory); + } + + var compressedTarType = await TryDetectCompressedTarAsync( + stream, + readerOptions, + startPosition, + cancellationToken + ) + .ConfigureAwait(false); + return compressedTarType is { } value ? CreateCompressedTarDetection(value) : null; + } + finally + { + stream.Seek(startPosition, SeekOrigin.Begin); + } + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask TryDetectCompressedTarAsync( + Stream stream, + ReaderOptions readerOptions, + long startPosition, + CancellationToken cancellationToken + ) + { + foreach (var wrapper in TarWrapper.Wrappers) + { +#if !SYNC_ONLY + cancellationToken.ThrowIfCancellationRequested(); +#endif + if (wrapper.CompressionType == CompressionType.None) + { + continue; + } + + stream.Seek(startPosition, SeekOrigin.Begin); + if (!await wrapper.IsMatchAsync(stream, cancellationToken).ConfigureAwait(false)) + { + continue; + } + + stream.Seek(startPosition, SeekOrigin.Begin); + if ( + await IsCompressedTarAsync( + stream, + readerOptions, + wrapper.CompressionType, + cancellationToken + ) + .ConfigureAwait(false) + ) + { + return wrapper.CompressionType; + } + } + + return null; + } +} diff --git a/src/SharpCompress/Archives/ArchiveFactory.Detection.cs b/src/SharpCompress/Archives/ArchiveFactory.Detection.cs index cf34e6e5..85721dbf 100644 --- a/src/SharpCompress/Archives/ArchiveFactory.Detection.cs +++ b/src/SharpCompress/Archives/ArchiveFactory.Detection.cs @@ -1,395 +1,13 @@ using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using SharpCompress.Archives.Tar; using SharpCompress.Common; +using SharpCompress.Detection; using SharpCompress.Factories; -using SharpCompress.IO; -using SharpCompress.Providers; using SharpCompress.Readers; namespace SharpCompress.Archives; public static partial class ArchiveFactory { - /// - /// Returns information about the archive at the given file path asynchronously, - /// or if the file is not a recognized archive. - /// - /// Path to the archive file. - /// Cancellation token. - public static async ValueTask GetArchiveInformationAsync( - string filePath, - CancellationToken cancellationToken = default - ) => - await GetArchiveInformationAsync(filePath, ReaderOptions.ForFilePath, cancellationToken) - .ConfigureAwait(false); - - /// - /// Returns information about the archive at the given file path asynchronously, - /// or if the file is not a recognized archive. - /// - /// Path to the archive file. - /// Options controlling archive detection. - /// Cancellation token. - public static async ValueTask GetArchiveInformationAsync( - string filePath, - ReaderOptions? readerOptions, - CancellationToken cancellationToken = default - ) - { - filePath.NotNullOrEmpty(nameof(filePath)); - using Stream stream = File.OpenRead(filePath); - return await GetArchiveInformationAsync( - stream, - readerOptions ?? ReaderOptions.ForFilePath, - cancellationToken - ) - .ConfigureAwait(false); - } - - /// - /// Returns information about the archive in the given stream asynchronously, - /// or if the stream is not a recognized archive. - /// - /// A readable and seekable stream positioned at the start of the archive. - /// Cancellation token. - public static async ValueTask GetArchiveInformationAsync( - Stream stream, - CancellationToken cancellationToken = default - ) => - await GetArchiveInformationAsync(stream, ReaderOptions.ForExternalStream, cancellationToken) - .ConfigureAwait(false); - - /// - /// Returns information about the archive in the given stream asynchronously, - /// or if the stream is not a recognized archive. - /// - /// A readable and seekable stream positioned at the start of the archive. - /// Options controlling archive detection. - /// Cancellation token. - public static async ValueTask GetArchiveInformationAsync( - Stream stream, - ReaderOptions? readerOptions, - CancellationToken cancellationToken = default - ) - { - stream.RequireReadable(); - stream.RequireSeekable(); - - var factory = await TryFindFactoryAsync( - stream, - readerOptions ?? ReaderOptions.ForExternalStream, - cancellationToken - ) - .ConfigureAwait(false); - return factory is null - ? null - : new ArchiveInformation(factory.KnownArchiveType, factory is IArchiveFactory); - } - - internal static ValueTask FindFactoryAsync( - string filePath, - CancellationToken cancellationToken = default - ) - where T : IFactory - { - filePath.NotNullOrEmpty(nameof(filePath)); - return FindFactoryAsync(new FileInfo(filePath), cancellationToken); - } - - internal static async ValueTask FindFactoryAsync( - FileInfo fileInfo, - CancellationToken cancellationToken = default - ) - where T : IFactory - { - fileInfo.NotNull(nameof(fileInfo)); - using Stream stream = fileInfo.OpenRead(); - return await FindFactoryAsync(stream, cancellationToken).ConfigureAwait(false); - } - - private static async ValueTask FindFactoryAsync( - FileInfo fileInfo, - ReaderOptions readerOptions, - CancellationToken cancellationToken - ) - where T : IFactory - { - fileInfo.NotNull(nameof(fileInfo)); - using Stream stream = fileInfo.OpenRead(); - return await FindFactoryAsync(stream, readerOptions, cancellationToken) - .ConfigureAwait(false); - } - - internal static async ValueTask FindFactoryAsync( - Stream stream, - CancellationToken cancellationToken = default - ) - where T : IFactory - { - stream.RequireReadable(); - stream.RequireSeekable(); - - // Use the shared async detection loop over all factories. If the matched factory - // implements T we return it; otherwise (or if nothing matched) we fall through - // to the same "unsupported format" exception that the original code produced, - // listing the T-typed factories as the hint for the caller. - return await FindFactoryAsync(stream, ReaderOptions.ForExternalStream, cancellationToken) - .ConfigureAwait(false); - } - - private static async ValueTask FindFactoryAsync( - Stream stream, - ReaderOptions readerOptions, - CancellationToken cancellationToken - ) - where T : IFactory - { - stream.RequireReadable(); - stream.RequireSeekable(); - - var factory = await TryFindFactoryAsync(stream, readerOptions, cancellationToken) - .ConfigureAwait(false); - if (factory is T typedFactory) - { - return typedFactory; - } - - var extensions = string.Join(", ", Factory.Factories.OfType().Select(item => item.Name)); - - throw new ArchiveOperationException( - $"Cannot determine compressed stream type. Supported Archive Formats: {extensions}" - ); - } - - /// - /// Async counterpart of the synchronous factory detection path. - /// Iterates all registered factories and returns the first one whose - /// recognises the stream, or . - /// Stream position is restored to its value at entry on both success and failure. - /// - private static async ValueTask TryFindFactoryAsync( - Stream stream, - ReaderOptions readerOptions, - CancellationToken cancellationToken - ) - { - var startPosition = stream.Position; - - foreach (var factory in Factory.Factories) - { - stream.Seek(startPosition, SeekOrigin.Begin); - var isArchive = await factory - .IsArchiveAsync(stream, readerOptions, cancellationToken) - .ConfigureAwait(false); - - if (isArchive) - { - stream.Seek(startPosition, SeekOrigin.Begin); - if ( - await IsCompressedTarAsync(stream, factory, readerOptions, cancellationToken) - .ConfigureAwait(false) - ) - { - continue; - } - - stream.Seek(startPosition, SeekOrigin.Begin); - return factory; - } - } - - stream.Seek(startPosition, SeekOrigin.Begin); - return null; - } - - /// - /// Returns information about the archive at the given file path, - /// or if the file is not a recognized archive. - /// - /// Path to the archive file. - public static ArchiveInformation? GetArchiveInformation(string filePath) => - GetArchiveInformation(filePath, ReaderOptions.ForFilePath); - - /// - /// Returns information about the archive at the given file path, - /// or if the file is not a recognized archive. - /// - /// Path to the archive file. - /// Options controlling archive detection. - public static ArchiveInformation? GetArchiveInformation( - string filePath, - ReaderOptions? readerOptions - ) - { - filePath.NotNullOrEmpty(nameof(filePath)); - using Stream stream = File.OpenRead(filePath); - return GetArchiveInformation(stream, readerOptions ?? ReaderOptions.ForFilePath); - } - - /// - /// Returns information about the archive in the given stream, - /// or if the stream is not a recognized archive. - /// - /// A readable and seekable stream positioned at the start of the archive. - public static ArchiveInformation? GetArchiveInformation(Stream stream) => - GetArchiveInformation(stream, ReaderOptions.ForExternalStream); - - /// - /// Returns information about the archive in the given stream, - /// or if the stream is not a recognized archive. - /// - /// A readable and seekable stream positioned at the start of the archive. - /// Options controlling archive detection. - public static ArchiveInformation? GetArchiveInformation( - Stream stream, - ReaderOptions? readerOptions - ) - { - stream.RequireReadable(); - stream.RequireSeekable(); - - var factory = TryFindFactory(stream, readerOptions ?? ReaderOptions.ForExternalStream); - return factory is null - ? null - : new ArchiveInformation(factory.KnownArchiveType, factory is IArchiveFactory); - } - - /// - /// Iterates all registered factories and returns the first one whose - /// recognises the stream, or . - /// Stream position is restored to its value at entry on both success and failure. - /// - /// - /// This is the shared, seekable-stream detection core used by - /// , , - /// and . - /// - /// uses a separate code path - /// based on rewindable buffering, which supports - /// non-seekable streams and is therefore not unified with this helper. - /// - /// - private static IFactory? TryFindFactory(Stream stream) => - TryFindFactory(stream, ReaderOptions.ForExternalStream); - - private static IFactory? TryFindFactory(Stream stream, ReaderOptions readerOptions) - { - var startPosition = stream.Position; - - foreach (var factory in Factory.Factories) - { - stream.Seek(startPosition, SeekOrigin.Begin); - var isArchive = factory.IsArchive(stream, readerOptions); - - if (isArchive) - { - stream.Seek(startPosition, SeekOrigin.Begin); - if (IsCompressedTar(stream, factory, readerOptions)) - { - continue; - } - - stream.Seek(startPosition, SeekOrigin.Begin); - return factory; - } - } - - stream.Seek(startPosition, SeekOrigin.Begin); - return null; - } - - private static bool IsCompressedTar( - Stream stream, - IFactory factory, - ReaderOptions readerOptions - ) => - GetCompressedTarType(factory) is { } compressionType - && IsCompressedTar(stream, readerOptions, compressionType); - - private static bool IsCompressedTar( - Stream stream, - ReaderOptions readerOptions, - CompressionType compressionType - ) - { - using var nonDisposingStream = SharpCompressStream.CreateNonDisposing(stream); - var testStream = - compressionType == CompressionType.GZip - ? readerOptions.Providers.CreateDecompressStream( - compressionType, - nonDisposingStream, - CompressionContext - .FromStream(nonDisposingStream) - .WithReaderOptions(readerOptions) - ) - : readerOptions.Providers.CreateDecompressStream( - compressionType, - nonDisposingStream - ); - - try - { - return TarArchive.IsTarFile(testStream); - } - finally - { - DisposeProbeStream(testStream); - } - } - - private static async ValueTask IsCompressedTarAsync( - Stream stream, - IFactory factory, - ReaderOptions readerOptions, - CancellationToken cancellationToken - ) => - GetCompressedTarType(factory) is { } compressionType - && await IsCompressedTarAsync(stream, readerOptions, compressionType, cancellationToken) - .ConfigureAwait(false); - - private static async ValueTask IsCompressedTarAsync( - Stream stream, - ReaderOptions readerOptions, - CompressionType compressionType, - CancellationToken cancellationToken - ) - { - using var nonDisposingStream = SharpCompressStream.CreateNonDisposing(stream); - var testStream = - compressionType == CompressionType.GZip - ? await readerOptions - .Providers.CreateDecompressStreamAsync( - compressionType, - nonDisposingStream, - CompressionContext - .FromStream(nonDisposingStream) - .WithReaderOptions(readerOptions), - cancellationToken - ) - .ConfigureAwait(false) - : await readerOptions - .Providers.CreateDecompressStreamAsync( - compressionType, - nonDisposingStream, - cancellationToken - ) - .ConfigureAwait(false); - - try - { - return await TarArchive - .IsTarFileAsync(testStream, cancellationToken) - .ConfigureAwait(false); - } - finally - { - DisposeProbeStream(testStream); - } - } - private static void DisposeProbeStream(Stream stream) { try @@ -411,4 +29,22 @@ public static partial class ArchiveFactory LzwFactory => CompressionType.Lzw, _ => null, }; + + private static ArchiveDetection CreateDetection(IFactory factory) + { + var supportedApis = ArchiveAccessMode.None; + if (factory is IArchiveFactory) + { + supportedApis |= ArchiveAccessMode.Archive; + } + if (factory is IReaderFactory) + { + supportedApis |= ArchiveAccessMode.Reader; + } + + return new ArchiveDetection(factory.KnownArchiveType, factory.Name, null, supportedApis); + } + + private static ArchiveDetection CreateCompressedTarDetection(CompressionType compressionType) => + new(ArchiveType.Tar, "Tar", compressionType, ArchiveAccessMode.Reader); } diff --git a/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs b/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs new file mode 100644 index 00000000..8ac74f5d --- /dev/null +++ b/src/SharpCompress/Archives/ArchiveFactory.Information.Async.cs @@ -0,0 +1,612 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives.Rar; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.Common.Ace.Headers; +using SharpCompress.Common.Rar; +using SharpCompress.Common.Zip; +using SharpCompress.Detection; +using SharpCompress.IO; +using SharpCompress.Readers; + +namespace SharpCompress.Archives; + +public static partial class ArchiveFactory +{ + /// + /// Collects metadata for the archive at the given file path. + /// + /// Path to the archive file. + /// Cancellation token. + /// Archive metadata, or when the file is not a supported archive. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + string filePath, + CancellationToken cancellationToken = default + ) => + await InspectArchiveAsync(filePath, ReaderOptions.ForFilePath, cancellationToken) + .ConfigureAwait(false); + + /// + /// Collects metadata for the archive at the given file path. + /// + /// Path to the archive file. + /// Options controlling archive inspection. + /// Cancellation token. + /// Archive metadata, or when the file is not a supported archive. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + string filePath, + ReaderOptions? readerOptions, + CancellationToken cancellationToken = default + ) + { + filePath.NotNullOrEmpty(nameof(filePath)); + var options = readerOptions ?? ReaderOptions.ForFilePath; + var detection = await DetectArchiveAsync(filePath, options, cancellationToken) + .ConfigureAwait(false); + if (detection is null) + { + return null; + } + if ((detection.SupportedApis & ArchiveAccessMode.Archive) != 0) + { + var fileInfos = GetArchiveFileParts(new FileInfo(filePath), options); + if (fileInfos.Length > 1) + { + return await InspectArchiveAsync(fileInfos, options, cancellationToken) + .ConfigureAwait(false); + } + } + + using Stream stream = File.OpenRead(filePath); + return await InspectArchiveAsync(stream, options, cancellationToken).ConfigureAwait(false); + } + + /// + /// Collects metadata for the archive in the given stream. + /// + /// A readable and seekable stream positioned at the start of the archive. + /// Cancellation token. + /// Archive metadata, or when the stream is not a supported archive. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + Stream stream, + CancellationToken cancellationToken = default + ) => + await InspectArchiveAsync(stream, ReaderOptions.ForExternalStream, cancellationToken) + .ConfigureAwait(false); + + /// + /// Collects metadata for the archive in the given stream. + /// + /// A readable and seekable stream positioned at the start of the archive. + /// Options controlling archive inspection. + /// Cancellation token. + /// Archive metadata, or when the stream is not a supported archive. + /// The supplied stream remains open and is restored to its original position. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + Stream stream, + ReaderOptions? readerOptions, + CancellationToken cancellationToken = default + ) + { + stream.RequireReadable(); + stream.RequireSeekable(); +#if !SYNC_ONLY + cancellationToken.ThrowIfCancellationRequested(); +#endif + + var options = readerOptions ?? ReaderOptions.ForExternalStream; + var startPosition = stream.Position; + var physicalSize = GetPhysicalSize(stream, startPosition); + + try + { + using var archiveStream = new ArchiveOffsetStream(stream); + var inspectionOptions = options with { LeaveStreamOpen = true }; + var detection = await TryDetectArchiveAsync( + archiveStream, + inspectionOptions, + cancellationToken + ) + .ConfigureAwait(false); + if (detection is null) + { + return null; + } + + if ((detection.SupportedApis & ArchiveAccessMode.Archive) != 0) + { +#if SYNC_ONLY + using var archive = OpenArchive(archiveStream, inspectionOptions); + return InspectOpenedArchive(archive, detection, physicalSize, 1); +#else + await using var archive = await OpenAsyncArchive( + archiveStream, + inspectionOptions, + cancellationToken + ) + .ConfigureAwait(false); + return await InspectOpenedArchiveAsync( + archive, + detection, + physicalSize, + 1, + cancellationToken + ) + .ConfigureAwait(false); +#endif + } + +#if SYNC_ONLY + var aceHeader = ReadAceHeader(archiveStream, detection, inspectionOptions); + using var reader = ReaderFactory.OpenReader(archiveStream, inspectionOptions); + return InspectOpenedReader(reader, detection, physicalSize, 1, aceHeader); +#else + var aceHeader = await ReadAceHeaderAsync( + archiveStream, + detection, + inspectionOptions, + cancellationToken + ) + .ConfigureAwait(false); + await using var reader = await ReaderFactory + .OpenAsyncReader(archiveStream, inspectionOptions, cancellationToken) + .ConfigureAwait(false); + return await InspectOpenedReaderAsync( + reader, + detection, + physicalSize, + 1, + aceHeader, + cancellationToken + ) + .ConfigureAwait(false); +#endif + } + catch (CryptographicException) when (string.IsNullOrEmpty(options.Password)) + { + var detection = await RedetectArchiveAsync( + stream, + startPosition, + options, + cancellationToken + ) + .ConfigureAwait(false); + return detection is null + ? null + : CreatePartialInformation( + detection, + physicalSize, + 1, + ArchiveInformationLimitations.EncryptedHeaders + ); + } + catch (MultipartStreamRequiredException) + { + var detection = await RedetectArchiveAsync( + stream, + startPosition, + options, + cancellationToken + ) + .ConfigureAwait(false); + return detection is null + ? null + : CreatePartialInformation( + detection, + physicalSize, + 1, + ArchiveInformationLimitations.MissingVolumes + ); + } + finally + { + stream.Seek(startPosition, SeekOrigin.Begin); + } + } + + /// + /// Collects metadata for an archive opened from multiple files. + /// + /// Archive source files in archive order. + /// Options controlling archive inspection. + /// Cancellation token. + /// Archive metadata, or when the files are not a supported archive. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + IReadOnlyList fileInfos, + ReaderOptions? readerOptions = null, + CancellationToken cancellationToken = default + ) + { + fileInfos.NotNull(nameof(fileInfos)); + if (fileInfos.Count == 0) + { + throw new ArchiveOperationException("No files to inspect"); + } + if (fileInfos.Count == 1) + { + return await InspectArchiveAsync( + fileInfos[0].FullName, + readerOptions, + cancellationToken + ) + .ConfigureAwait(false); + } + + var options = readerOptions ?? ReaderOptions.ForFilePath; + var detection = await DetectArchiveAsync(fileInfos[0].FullName, options, cancellationToken) + .ConfigureAwait(false); + if (detection is null) + { + return null; + } + if ((detection.SupportedApis & ArchiveAccessMode.Archive) == 0) + { + throw new NotSupportedException( + "Inspecting multiple source files is supported only for formats with an Archive API." + ); + } + + var physicalSize = GetPhysicalSize(fileInfos); + try + { +#if SYNC_ONLY + using var archive = OpenArchive(fileInfos, options); + return InspectOpenedArchive(archive, detection, physicalSize, fileInfos.Count); +#else + await using var archive = await OpenAsyncArchive(fileInfos, options, cancellationToken) + .ConfigureAwait(false); + return await InspectOpenedArchiveAsync( + archive, + detection, + physicalSize, + fileInfos.Count, + cancellationToken + ) + .ConfigureAwait(false); +#endif + } + catch (CryptographicException) when (string.IsNullOrEmpty(options.Password)) + { + return CreatePartialInformation( + detection, + physicalSize, + fileInfos.Count, + ArchiveInformationLimitations.EncryptedHeaders + ); + } + } + + /// + /// Collects metadata for an archive opened from multiple streams. + /// + /// Archive source streams in archive order. + /// Options controlling archive inspection. + /// Cancellation token. + /// Archive metadata, or when the streams are not a supported archive. + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public static async ValueTask InspectArchiveAsync( + IReadOnlyList streams, + ReaderOptions? readerOptions = null, + CancellationToken cancellationToken = default + ) + { + streams.NotNull(nameof(streams)); + if (streams.Count == 0) + { + throw new ArchiveOperationException("No streams to inspect"); + } + if (streams.Count == 1) + { + return await InspectArchiveAsync(streams[0], readerOptions, cancellationToken) + .ConfigureAwait(false); + } + + var options = readerOptions ?? ReaderOptions.ForExternalStream; + var startPositions = streams.Select(stream => stream.Position).ToArray(); + var physicalSize = GetPhysicalSize(streams, startPositions); + + try + { + using var firstArchiveStream = new ArchiveOffsetStream(streams[0]); + var inspectionOptions = options with { LeaveStreamOpen = true }; + var detection = await TryDetectArchiveAsync( + firstArchiveStream, + inspectionOptions, + cancellationToken + ) + .ConfigureAwait(false); + if (detection is null) + { + return null; + } + if ((detection.SupportedApis & ArchiveAccessMode.Archive) == 0) + { + throw new NotSupportedException( + "Inspecting multiple source streams is supported only for formats with an Archive API." + ); + } + + var archiveStreams = new List { firstArchiveStream }; + archiveStreams.AddRange( + streams.Skip(1).Select(stream => new ArchiveOffsetStream(stream)) + ); +#if SYNC_ONLY + using var archive = OpenArchive(archiveStreams, inspectionOptions); + return InspectOpenedArchive(archive, detection, physicalSize, streams.Count); +#else + await using var archive = await OpenAsyncArchive( + archiveStreams, + inspectionOptions, + cancellationToken + ) + .ConfigureAwait(false); + return await InspectOpenedArchiveAsync( + archive, + detection, + physicalSize, + streams.Count, + cancellationToken + ) + .ConfigureAwait(false); +#endif + } + catch (CryptographicException) when (string.IsNullOrEmpty(options.Password)) + { + var detection = await RedetectArchiveAsync( + streams[0], + startPositions[0], + options, + cancellationToken + ) + .ConfigureAwait(false); + return detection is null + ? null + : CreatePartialInformation( + detection, + physicalSize, + streams.Count, + ArchiveInformationLimitations.EncryptedHeaders + ); + } + finally + { + for (var i = 0; i < streams.Count; i++) + { + streams[i].Seek(startPositions[i], SeekOrigin.Begin); + } + } + } + + private static async ValueTask InspectOpenedArchiveAsync( + IAsyncArchive archive, + ArchiveDetection detection, + long? physicalSize, + int physicalPartCount, + CancellationToken cancellationToken + ) + { + var entries = new List(); + await foreach ( + var entry in archive + .EntriesAsync.WithCancellation(cancellationToken) + .ConfigureAwait(false) + ) + { + cancellationToken.ThrowIfCancellationRequested(); + entries.Add(entry); + } + + var volumes = new List(); + await foreach ( + var volume in archive + .VolumesAsync.WithCancellation(cancellationToken) + .ConfigureAwait(false) + ) + { + cancellationToken.ThrowIfCancellationRequested(); + volumes.Add(volume); + } + + var entryArray = entries.ToArray(); + var volumeArray = volumes.ToArray(); + var (zipInformation, deferredSizeEntryCount) = await GetZipInformationAsync( + archive.Type == ArchiveType.Zip, + entryArray, + cancellationToken + ) + .ConfigureAwait(false); + var (isSolid, solidStreamCount) = await GetSolidInformationAsync(archive, entryArray) + .ConfigureAwait(false); + var isComplete = await archive.IsCompleteAsync().ConfigureAwait(false); + var limitations = isComplete + ? ArchiveInformationLimitations.None + : ArchiveInformationLimitations.MissingVolumes; + if (archive.Type == ArchiveType.GZip) + { + limitations |= ArchiveInformationLimitations.UnavailableMetadata; + } + var hasEncryptedHeaders = volumeArray + .OfType() + .Any(volume => volume.IsHeaderEncrypted); + var hasEncryptedEntries = entryArray.Any(entry => entry.IsEncrypted); + + return new ArchiveInformation( + detection, + GetStatus(limitations), + limitations, + GetFormatVersion(volumeArray), + entryArray.LongLength, + deferredSizeEntryCount, + physicalSize, + isComplete ? GetCompressedPayloadSize(archive, entryArray) : null, + isComplete && archive.Type != ArchiveType.GZip + ? entryArray.Aggregate(0L, (total, entry) => total + entry.Size) + : null, + isSolid, + solidStreamCount, + GetEncryptionScope(hasEncryptedHeaders, hasEncryptedEntries), + hasEncryptedHeaders || hasEncryptedEntries, + GetIsMultiVolume(archive.Type, volumeArray), + physicalPartCount, + volumeArray.Length, + isComplete, + GetArchiveComment(volumeArray), + zipInformation + ); + } + + private static async ValueTask InspectOpenedReaderAsync( + IAsyncReader reader, + ArchiveDetection detection, + long? physicalSize, + int physicalPartCount, + AceMainHeader? aceHeader, + CancellationToken cancellationToken + ) + { + var inspection = new ReaderInspection(detection); + while (await reader.MoveToNextEntryAsync(cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + inspection.Add(reader.Entry); + } + + var isSolid = aceHeader?.IsSolid ?? false; + var isMultiVolume = aceHeader?.IsMultiVolume ?? false; + var formatVersion = aceHeader is null ? null : $"ACE {aceHeader.AceVersion / 10.0:0.0}"; + return inspection.CreateInformation( + physicalSize, + physicalPartCount, + isSolid, + isSolid && inspection.EntryCount > 1 ? 1 : 0, + isMultiVolume, + formatVersion + ); + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask ReadAceHeaderAsync( + Stream stream, + ArchiveDetection detection, + ReaderOptions options, + CancellationToken cancellationToken + ) + { + if (detection.ContainerType != ArchiveType.Ace) + { + return null; + } + + try + { + stream.Position = 0; + return await new AceMainHeader(options.ArchiveEncoding) + .ReadAsync(stream, cancellationToken) + .ConfigureAwait(false) as AceMainHeader; + } + finally + { + stream.Position = 0; + } + } + + private static async ValueTask<(bool IsSolid, long SolidStreamCount)> GetSolidInformationAsync( + IAsyncArchive archive, + IReadOnlyCollection entries + ) + { + if (archive.Type == ArchiveType.SevenZip) + { + var solidStreamCount = entries + .OfType() + .Where(entry => !entry.IsDirectory && entry.FilePart.Folder is not null) + .GroupBy(entry => entry.FilePart.Folder) + .LongCount(group => group.Skip(1).Any()); + return (solidStreamCount > 0, solidStreamCount); + } + + if (archive.Type == ArchiveType.Rar) + { + return ( + await archive.IsSolidAsync().ConfigureAwait(false), + CountRarSolidStreams(entries) + ); + } + + return (false, 0); + } + + private static long? GetCompressedPayloadSize( + IAsyncArchive archive, + IReadOnlyCollection entries + ) => + archive.Type switch + { + ArchiveType.GZip => null, + ArchiveType.SevenZip when archive is SevenZipArchive sevenZipArchive => + sevenZipArchive.TotalSize, + _ => entries.Aggregate(0L, (total, entry) => total + entry.CompressedSize), + }; + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask<( + ZipArchiveInformation? Information, + long DeferredSizeEntryCount + )> GetZipInformationAsync( + bool isZipArchive, + IEnumerable entries, + CancellationToken cancellationToken + ) + { + if (!isZipArchive) + { + return (null, 0); + } + + long deferredSizeEntryCount = 0; + foreach (var entry in entries.OfType()) + { +#if !SYNC_ONLY + cancellationToken.ThrowIfCancellationRequested(); +#endif + var filePart = entry.Parts.OfType().Single(); + if (!filePart.HasDeferredSizes) + { + continue; + } + + var localHeader = await filePart.GetRawLocalHeaderAsync().ConfigureAwait(false); + if (localHeader.CompressedSize == 0 && localHeader.UncompressedSize == 0) + { + deferredSizeEntryCount++; + } + } + + return (new ZipArchiveInformation(deferredSizeEntryCount > 0), deferredSizeEntryCount); + } + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + private static async ValueTask RedetectArchiveAsync( + Stream stream, + long startPosition, + ReaderOptions options, + CancellationToken cancellationToken + ) + { + stream.Seek(startPosition, SeekOrigin.Begin); + using var archiveStream = new ArchiveOffsetStream(stream); + return await TryDetectArchiveAsync(archiveStream, options, cancellationToken) + .ConfigureAwait(false); + } +} diff --git a/src/SharpCompress/Archives/ArchiveFactory.Information.cs b/src/SharpCompress/Archives/ArchiveFactory.Information.cs new file mode 100644 index 00000000..4b12fe8d --- /dev/null +++ b/src/SharpCompress/Archives/ArchiveFactory.Information.cs @@ -0,0 +1,403 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using SharpCompress.Archives.Rar; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.Common.Rar; +using SharpCompress.Common.Zip; +using SharpCompress.Common.Zip.Headers; +using SharpCompress.Detection; +using SharpCompress.Readers; +using AceMainHeader = SharpCompress.Common.Ace.Headers.AceMainHeader; + +namespace SharpCompress.Archives; + +public static partial class ArchiveFactory +{ + private static ArchiveInformation InspectOpenedArchive( + IArchive archive, + ArchiveDetection detection, + long? physicalSize, + int physicalPartCount + ) + { + var entries = archive.Entries.ToArray(); + var volumes = archive.Volumes.ToArray(); + var (zipInformation, deferredSizeEntryCount) = GetZipInformation( + archive.Type == ArchiveType.Zip, + entries + ); + var (isSolid, solidStreamCount) = GetSolidInformation(archive, entries); + var isMultiVolume = GetIsMultiVolume(archive.Type, volumes); + var comment = GetArchiveComment(volumes); + var hasEncryptedHeaders = volumes + .OfType() + .Any(volume => volume.IsHeaderEncrypted); + var hasEncryptedEntries = entries.Any(entry => entry.IsEncrypted); + var isComplete = archive.IsComplete; + var limitations = isComplete + ? ArchiveInformationLimitations.None + : ArchiveInformationLimitations.MissingVolumes; + if (archive.Type == ArchiveType.GZip) + { + limitations |= ArchiveInformationLimitations.UnavailableMetadata; + } + + return new ArchiveInformation( + detection, + GetStatus(limitations), + limitations, + GetFormatVersion(volumes), + entries.LongLength, + deferredSizeEntryCount, + physicalSize, + isComplete ? GetCompressedPayloadSize(archive, entries) : null, + isComplete && archive.Type != ArchiveType.GZip + ? entries.Aggregate(0L, (total, entry) => total + entry.Size) + : null, + isSolid, + solidStreamCount, + GetEncryptionScope(hasEncryptedHeaders, hasEncryptedEntries), + hasEncryptedHeaders || hasEncryptedEntries, + isMultiVolume, + physicalPartCount, + volumes.Length, + isComplete, + comment, + zipInformation + ); + } + + private static ArchiveInformation InspectOpenedReader( + IReader reader, + ArchiveDetection detection, + long? physicalSize, + int physicalPartCount, + AceMainHeader? aceHeader + ) + { + var inspection = new ReaderInspection(detection); + while (reader.MoveToNextEntry()) + { + inspection.Add(reader.Entry); + } + + var isSolid = aceHeader?.IsSolid ?? false; + var isMultiVolume = aceHeader?.IsMultiVolume ?? false; + var formatVersion = aceHeader is null ? null : $"ACE {aceHeader.AceVersion / 10.0:0.0}"; + return inspection.CreateInformation( + physicalSize, + physicalPartCount, + isSolid, + isSolid && inspection.EntryCount > 1 ? 1 : 0, + isMultiVolume, + formatVersion + ); + } + + private static ArchiveInformation CreatePartialInformation( + ArchiveDetection detection, + long? physicalSize, + int physicalPartCount, + ArchiveInformationLimitations limitations + ) => + new( + detection, + ArchiveInformationStatus.Partial, + limitations, + null, + null, + null, + physicalSize, + null, + null, + null, + null, + limitations.HasFlag(ArchiveInformationLimitations.EncryptedHeaders) + ? ArchiveEncryptionScope.Headers + : null, + limitations.HasFlag(ArchiveInformationLimitations.EncryptedHeaders) ? true : null, + null, + physicalPartCount, + null, + null, + null, + null + ); + + private static ArchiveInformationStatus GetStatus(ArchiveInformationLimitations limitations) => + limitations == ArchiveInformationLimitations.None + ? ArchiveInformationStatus.Complete + : ArchiveInformationStatus.Partial; + + private static (bool IsSolid, long SolidStreamCount) GetSolidInformation( + IArchive archive, + IReadOnlyCollection entries + ) + { + if (archive.Type == ArchiveType.SevenZip) + { + var solidStreamCount = entries + .OfType() + .Where(entry => !entry.IsDirectory && entry.FilePart.Folder is not null) + .GroupBy(entry => entry.FilePart.Folder) + .LongCount(group => group.Skip(1).Any()); + return (solidStreamCount > 0, solidStreamCount); + } + + if (archive.Type == ArchiveType.Rar) + { + var solidStreamCount = CountRarSolidStreams(entries); + return (archive.IsSolid, solidStreamCount); + } + + return (false, 0); + } + + private static long CountRarSolidStreams(IEnumerable entries) + { + var wasSolid = false; + long count = 0; + foreach (var entry in entries.Where(entry => !entry.IsDirectory)) + { + if (entry.IsSolid && !wasSolid) + { + count++; + } + wasSolid = entry.IsSolid; + } + return count; + } + + private static bool GetIsMultiVolume(ArchiveType type, IVolume[] volumes) => + type == ArchiveType.Rar + ? volumes.OfType().Any(volume => volume.IsMultiVolume) + : volumes.Length > 1; + + private static string? GetArchiveComment(IEnumerable volumes) => + volumes.OfType().LastOrDefault()?.Comment + ?? volumes + .OfType() + .Select(volume => volume.Comment) + .FirstOrDefault(comment => comment is not null); + + private static string? GetFormatVersion(IEnumerable volumes) + { + var rarVolume = volumes.OfType().FirstOrDefault(); + return rarVolume is null ? null : $"RAR {rarVolume.MinVersion}-{rarVolume.MaxVersion}"; + } + + private static ArchiveEncryptionScope GetEncryptionScope( + bool hasEncryptedHeaders, + bool hasEncryptedEntries + ) + { + var encryption = ArchiveEncryptionScope.None; + if (hasEncryptedHeaders) + { + encryption |= ArchiveEncryptionScope.Headers; + } + if (hasEncryptedEntries) + { + encryption |= ArchiveEncryptionScope.EntryData; + } + return encryption; + } + + private static long? GetCompressedPayloadSize( + IArchive archive, + IReadOnlyCollection entries + ) => + archive.Type switch + { + ArchiveType.GZip => null, + ArchiveType.SevenZip when archive is SevenZipArchive sevenZipArchive => + sevenZipArchive.TotalSize, + _ => entries.Aggregate(0L, (total, entry) => total + entry.CompressedSize), + }; + + private static long? GetPhysicalSize(Stream stream, long startPosition) + { + try + { + return stream.Length - startPosition; + } + catch (NotSupportedException) + { + return null; + } + } + + private static FileInfo[] GetArchiveFileParts(FileInfo firstPart, ReaderOptions options) + { + using Stream stream = firstPart.OpenRead(); + var factory = TryFindFactory(stream, options); + if (factory is null) + { + return [firstPart]; + } + + var parts = new List { firstPart }; + for (var index = 1; factory.GetFilePart(index, firstPart) is { } part; index++) + { + parts.Add(part); + } + return parts.ToArray(); + } + + private static long? GetPhysicalSize(IEnumerable fileInfos) + { + try + { + return fileInfos.Aggregate(0L, (total, fileInfo) => total + fileInfo.Length); + } + catch (IOException) + { + return null; + } + } + + private static long? GetPhysicalSize(IReadOnlyList streams, long[] startPositions) + { + try + { + long total = 0; + for (var i = 0; i < streams.Count; i++) + { + total += streams[i].Length - startPositions[i]; + } + return total; + } + catch (NotSupportedException) + { + return null; + } + } + + private sealed class ReaderInspection + { + private readonly ArchiveDetection detection; + private long? compressedPayloadSize = 0; + private long? uncompressedPayloadSize = 0; + private long deferredSizeEntryCount; + private long entriesWithUnknownSizeCount; + private bool isEncrypted; + private ArchiveInformationLimitations limitations; + + public ReaderInspection(ArchiveDetection detection) + { + this.detection = detection; + if ( + detection.ContainerType == ArchiveType.Lzw + || ( + detection.ContainerType == ArchiveType.Tar + && detection.OuterCompressionType is not null + ) + ) + { + compressedPayloadSize = null; + limitations |= ArchiveInformationLimitations.UnavailableMetadata; + } + if (detection.ContainerType == ArchiveType.Lzw) + { + uncompressedPayloadSize = null; + } + } + + public long EntryCount { get; private set; } + + public void Add(IEntry entry) + { + EntryCount++; + isEncrypted |= entry.IsEncrypted; + + var hasDeferredSizes = HasDeferredSizes(entry); + if (hasDeferredSizes) + { + deferredSizeEntryCount++; + } + + if ( + detection.ContainerType == ArchiveType.Lzw + || hasDeferredSizes + || !TryGetSize(entry, out var size) + ) + { + entriesWithUnknownSizeCount++; + uncompressedPayloadSize = null; + limitations |= ArchiveInformationLimitations.UnavailableMetadata; + } + else if (uncompressedPayloadSize is { } totalSize) + { + uncompressedPayloadSize = totalSize + size; + } + + if (compressedPayloadSize is { } totalCompressedSize) + { + compressedPayloadSize = totalCompressedSize + entry.CompressedSize; + } + } + + public ArchiveInformation CreateInformation( + long? physicalSize, + int physicalPartCount, + bool isSolid, + long solidStreamCount, + bool isMultiVolume, + string? formatVersion + ) => + new( + detection, + GetStatus(limitations), + limitations, + formatVersion, + EntryCount, + entriesWithUnknownSizeCount, + physicalSize, + compressedPayloadSize, + uncompressedPayloadSize, + isSolid, + solidStreamCount, + isEncrypted ? ArchiveEncryptionScope.EntryData : ArchiveEncryptionScope.None, + isEncrypted, + isMultiVolume, + physicalPartCount, + 1, + true, + null, + detection.ContainerType == ArchiveType.Zip + ? new ZipArchiveInformation(deferredSizeEntryCount > 0) + : null + ); + + private static bool TryGetSize(IEntry entry, out long size) + { + try + { + size = entry.Size; + return size >= 0; + } + catch (NotImplementedException) + { + size = 0; + return false; + } + } + + private static bool HasDeferredSizes(IEntry entry) => + entry is ZipEntry zipEntry + && zipEntry + .Parts.OfType() + .Any(part => + FlagUtility.HasFlag( + part.Header.Flags, + SharpCompress.Common.Zip.Headers.HeaderFlags.UsePostDataDescriptor + ) + && part.Header.CompressedSize == 0 + && part.Header.UncompressedSize == 0 + ); + } +} diff --git a/src/SharpCompress/Archives/ArchiveFactory.cs b/src/SharpCompress/Archives/ArchiveFactory.cs index 63b63954..efff5604 100644 --- a/src/SharpCompress/Archives/ArchiveFactory.cs +++ b/src/SharpCompress/Archives/ArchiveFactory.cs @@ -123,37 +123,6 @@ public static partial class ArchiveFactory public static T FindFactory(Stream stream) where T : IFactory => FindFactory(stream, ReaderOptions.ForExternalStream); - private static T FindFactory(FileInfo fileInfo, ReaderOptions readerOptions) - where T : IFactory - { - fileInfo.NotNull(nameof(fileInfo)); - using Stream stream = fileInfo.OpenRead(); - return FindFactory(stream, readerOptions); - } - - private static T FindFactory(Stream stream, ReaderOptions readerOptions) - where T : IFactory - { - stream.RequireReadable(); - stream.RequireSeekable(); - - // Use the shared detection loop over all factories. If the matched factory - // implements T we return it; otherwise (or if nothing matched) we fall through - // to the same "unsupported format" exception that the original code produced, - // listing the T-typed factories as the hint for the caller. - var factory = TryFindFactory(stream, readerOptions); - if (factory is T typedFactory) - { - return typedFactory; - } - - var extensions = string.Join(", ", Factory.Factories.OfType().Select(item => item.Name)); - - throw new ArchiveOperationException( - $"Cannot determine compressed stream type. Supported Archive Formats: {extensions}" - ); - } - public static bool IsArchive(string filePath, out ArchiveType? type) { return IsArchive(filePath, ReaderOptions.ForFilePath, out type); diff --git a/src/SharpCompress/Archives/ArchiveInformation.cs b/src/SharpCompress/Archives/ArchiveInformation.cs deleted file mode 100644 index 75d3cb5a..00000000 --- a/src/SharpCompress/Archives/ArchiveInformation.cs +++ /dev/null @@ -1,38 +0,0 @@ -using SharpCompress.Common; - -namespace SharpCompress.Archives; - -/// -/// Contains information about a detected archive, including its type and supported capabilities. -/// -/// -/// Use or -/// -/// to obtain an instance of this record. -/// -public record ArchiveInformation -{ - /// - /// The type of archive detected, or when the format is not a registered well-known type. - /// - public ArchiveType? Type { get; set; } - - /// - /// when this archive format supports random access via the API, - /// meaning the full file listing can be retrieved without decompressing the entire archive. - /// when only the API is available, - /// which reads entries sequentially and can only report per-entry progress. - /// - public bool SupportsRandomAccess { get; set; } - - /// - /// Creates a new archive information instance. - /// - /// The detected archive type. - /// Whether the detected format supports random access. - public ArchiveInformation(ArchiveType? type, bool supportsRandomAccess) - { - Type = type; - SupportsRandomAccess = supportsRandomAccess; - } -} diff --git a/src/SharpCompress/Archives/GZip/GZipArchive.Async.cs b/src/SharpCompress/Archives/GZip/GZipArchive.Async.cs index 49f03937..7b2cda51 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchive.Async.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchive.Async.cs @@ -15,9 +15,11 @@ namespace SharpCompress.Archives.GZip; public partial class GZipArchive { + [Zomp.SyncMethodGenerator.CreateSyncVersion] public ValueTask SaveToAsync(string filePath, CancellationToken cancellationToken = default) => SaveToAsync(new FileInfo(filePath), cancellationToken); + [Zomp.SyncMethodGenerator.CreateSyncVersion] public async ValueTask SaveToAsync( FileInfo fileInfo, CancellationToken cancellationToken = default @@ -28,6 +30,7 @@ public partial class GZipArchive .ConfigureAwait(false); } + [Zomp.SyncMethodGenerator.CreateSyncVersion] protected override async ValueTask SaveToAsync( Stream stream, GZipWriterOptions options, @@ -54,6 +57,7 @@ public partial class GZipArchive .WriteAsync( entry.Key.NotNull("Entry Key is null"), entryStream, + entry.LastModifiedTime, cancellationToken ) .ConfigureAwait(false); @@ -65,7 +69,12 @@ public partial class GZipArchive .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); await writer - .WriteAsync(entry.Key.NotNull("Entry Key is null"), entryStream, cancellationToken) + .WriteAsync( + entry.Key.NotNull("Entry Key is null"), + entryStream, + entry.LastModifiedTime, + cancellationToken + ) .ConfigureAwait(false); } } diff --git a/src/SharpCompress/Archives/GZip/GZipArchive.Factory.cs b/src/SharpCompress/Archives/GZip/GZipArchive.Factory.cs index 2f5e5e33..5f4e7726 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchive.Factory.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchive.Factory.cs @@ -182,7 +182,12 @@ public partial class GZipArchive var header = ArrayPool.Shared.Rent(10); try { - await stream.ReadFullyAsync(header, 0, 10, cancellationToken).ConfigureAwait(false); + if ( + !await stream.ReadFullyAsync(header, 0, 10, cancellationToken).ConfigureAwait(false) + ) + { + return false; + } if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8) { diff --git a/src/SharpCompress/Archives/GZip/GZipArchive.cs b/src/SharpCompress/Archives/GZip/GZipArchive.cs index e47bf7c6..ab3a009d 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchive.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchive.cs @@ -8,7 +8,6 @@ using SharpCompress.Common.Options; using SharpCompress.IO; using SharpCompress.Readers; using SharpCompress.Readers.GZip; -using SharpCompress.Writers; using SharpCompress.Writers.GZip; namespace SharpCompress.Archives.GZip; @@ -28,14 +27,6 @@ public partial class GZipArchive return sourceStream.Streams.Select(a => new GZipVolume(a, ReaderOptions, 0)); } - public void SaveTo(string filePath) => SaveTo(new FileInfo(filePath)); - - public void SaveTo(FileInfo fileInfo) - { - using var stream = fileInfo.Open(FileMode.Create, FileAccess.Write); - SaveTo(stream, new GZipWriterOptions(CompressionType.GZip)); - } - protected override GZipArchiveEntry CreateEntryInternal( string key, Stream source, @@ -54,31 +45,9 @@ public partial class GZipArchive protected override GZipArchiveEntry CreateDirectoryEntry(string key, DateTime? modified) => throw new NotSupportedException("GZip archives do not support directory entries."); - protected override void SaveTo( - Stream stream, - GZipWriterOptions options, - IEnumerable oldEntries, - IEnumerable newEntries - ) - { - if (Entries.Count > 1) - { - throw new InvalidFormatException("Only one entry is allowed in a GZip Archive"); - } - using var writer = new GZipWriter(stream, options); - foreach (var entry in oldEntries.Concat(newEntries).Where(x => !x.IsDirectory)) - { - using var entryStream = entry.OpenEntryStream(); - writer.Write( - entry.Key.NotNull("Entry Key is null"), - entryStream, - entry.LastModifiedTime - ); - } - } - protected override IEnumerable LoadEntries(IEnumerable volumes) { + // Kept hand-written: the generator cannot map IAsyncEnumerable.SingleAsync to LINQ Single. var stream = volumes.Single().Stream; yield return new GZipArchiveEntry( this, diff --git a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs index 9d975788..4c3bed6c 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs @@ -7,23 +7,12 @@ using SharpCompress.Common.Options; namespace SharpCompress.Archives.GZip; -public class GZipArchiveEntry : GZipEntry, IArchiveEntry +public partial class GZipArchiveEntry : GZipEntry, IArchiveEntry { internal GZipArchiveEntry(GZipArchive archive, GZipFilePart? part, IReaderOptions readerOptions) : base(part, readerOptions) => Archive = archive; - public virtual Stream OpenEntryStream() - { - //this is to reset the stream to be read multiple times - var part = (GZipFilePart)Parts.Single(); - var rawStream = part.GetRawStream(); - if (rawStream.CanSeek && rawStream.Position != part.EntryStartPosition) - { - rawStream.Position = part.EntryStartPosition; - } - return Parts.Single().GetCompressedStream().NotNull(); - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public virtual async ValueTask OpenEntryStreamAsync( CancellationToken cancellationToken = default ) diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 6b6b96b7..31b4596a 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -71,13 +71,14 @@ public static partial class IArchiveEntryExtensions throw new ExtractionException("Entry is a file directory and cannot be extracted."); } -#if LEGACY_DOTNET - using var entryStream = await archiveEntry +#if SYNC_ONLY + using var entryStream = archiveEntry.OpenEntryStream(); +#else + var entryStream = await archiveEntry .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); -#else - await using var entryStream = await archiveEntry - .OpenEntryStreamAsync(cancellationToken) + await using var entryStreamScope = entryStream + .DisposeAsyncScope() .ConfigureAwait(false); #endif var checkedStream = options is null diff --git a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.Async.cs b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.Async.cs index ce00f20d..8cacfa43 100644 --- a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.Async.cs +++ b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.Async.cs @@ -46,7 +46,14 @@ public partial class SevenZipArchive var file = _database._files[i]; entries[i] = new SevenZipArchiveEntry( this, - new SevenZipFilePart(stream, _database, i, file, ReaderOptions.ArchiveEncoding), + new SevenZipFilePart( + stream, + _database, + i, + file, + ReaderOptions.ArchiveEncoding, + ReaderOptions.EnableParallelism + ), ReaderOptions ); } diff --git a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs index 27822313..c414065e 100644 --- a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs +++ b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs @@ -54,7 +54,8 @@ public partial class SevenZipArchive : AbstractArchive value == Path.DirectorySeparatorChar || value == Path.AltDirectorySeparatorChar; diff --git a/src/SharpCompress/Common/EntryStream.Async.cs b/src/SharpCompress/Common/EntryStream.Async.cs index e284c33f..87f52782 100644 --- a/src/SharpCompress/Common/EntryStream.Async.cs +++ b/src/SharpCompress/Common/EntryStream.Async.cs @@ -18,7 +18,6 @@ public partial class EntryStream _completed = true; } -#if !LEGACY_DOTNET public override async ValueTask DisposeAsync() { if (_isDisposed) @@ -44,9 +43,8 @@ public partial class EntryStream } } await base.DisposeAsync().ConfigureAwait(false); - await _stream.DisposeAsync().ConfigureAwait(false); + await _stream.DisposeAsyncCompat().ConfigureAwait(false); } -#endif [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index 3c427c4f..bf835b93 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -8,7 +8,7 @@ using SharpCompress.Readers; namespace SharpCompress.Common; -public partial class EntryStream : Stream +public partial class EntryStream : AsyncDisposableStream { private readonly IReader _reader; private readonly Stream _stream; @@ -30,18 +30,7 @@ public partial class EntryStream : Stream _isDisposed = true; if (!(_completed || _reader.Cancelled)) { - if (Utility.UseSyncOverAsyncDispose()) - { -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits -#pragma warning disable CA2012 - SkipEntryAsync().GetAwaiter().GetResult(); -#pragma warning restore CA2012 -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - else - { - SkipEntry(); - } + SkipEntry(); } //Need a safe standard approach to this - it's okay for compression to overreads. Handling needs to be standardised diff --git a/src/SharpCompress/Common/ExtractionOptions.cs b/src/SharpCompress/Common/ExtractionOptions.cs index 04b33416..ce58a529 100644 --- a/src/SharpCompress/Common/ExtractionOptions.cs +++ b/src/SharpCompress/Common/ExtractionOptions.cs @@ -60,6 +60,8 @@ public sealed record ExtractionOptions : IExtractionOptions /// /// Breaking change: Changed from field to property in version 0.40.0. /// If no handler is provided, symbolic links are silently skipped during extraction. + /// Directory extraction rejects link targets outside the destination directory and does not + /// follow symbolic links or reparse points in later entry paths. /// public Action? SymbolicLinkHandler { get; set; } diff --git a/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs index 512d7cc2..accf1db4 100644 --- a/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs +++ b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs @@ -7,7 +7,7 @@ using SharpCompress.Crypto; namespace SharpCompress.Common.GZip; -internal sealed class GZipChecksumValidationStream : Stream +internal sealed partial class GZipChecksumValidationStream : Stream { private readonly Stream _source; private readonly Stream _rawStream; @@ -46,23 +46,38 @@ internal sealed class GZipChecksumValidationStream : Stream set => throw new NotSupportedException(); } - public override void Flush() => _source.Flush(); - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override Task FlushAsync(CancellationToken cancellationToken) => _source.FlushAsync(cancellationToken); - public override int Read(byte[] buffer, int offset, int count) + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) { - var read = _source.Read(buffer, offset, count); + var read = await _source + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); return read; } #if !LEGACY_DOTNET - public override int Read(Span buffer) + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) { - var read = _source.Read(buffer); + var read = await _source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); +#if SYNC_ONLY UpdateAndValidateAtEof(buffer[..read], read); +#else + UpdateAndValidateAtEof(buffer.Span[..read], read); +#endif return read; } #endif @@ -83,32 +98,6 @@ internal sealed class GZipChecksumValidationStream : Stream return value; } - public override async Task ReadAsync( - byte[] buffer, - int offset, - int count, - CancellationToken cancellationToken - ) - { - var read = await _source - .ReadAsync(buffer, offset, count, cancellationToken) - .ConfigureAwait(false); - UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); - return read; - } - -#if !LEGACY_DOTNET - public override async ValueTask ReadAsync( - Memory buffer, - CancellationToken cancellationToken = default - ) - { - var read = await _source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - UpdateAndValidateAtEof(buffer.Span[..read], read); - return read; - } -#endif - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); diff --git a/src/SharpCompress/Common/GZip/GZipEntry.Async.cs b/src/SharpCompress/Common/GZip/GZipEntry.Async.cs index 7bb5f983..a19caa71 100644 --- a/src/SharpCompress/Common/GZip/GZipEntry.Async.cs +++ b/src/SharpCompress/Common/GZip/GZipEntry.Async.cs @@ -6,6 +6,7 @@ namespace SharpCompress.Common.GZip; public partial class GZipEntry { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal static async IAsyncEnumerable GetEntriesAsync( Stream stream, ReaderOptions options diff --git a/src/SharpCompress/Common/GZip/GZipEntry.cs b/src/SharpCompress/Common/GZip/GZipEntry.cs index 48f08d46..e359198e 100644 --- a/src/SharpCompress/Common/GZip/GZipEntry.cs +++ b/src/SharpCompress/Common/GZip/GZipEntry.cs @@ -46,14 +46,4 @@ public partial class GZipEntry : Entry public override bool IsSplitAfter => false; internal override IEnumerable Parts => _filePart.Empty(); - - internal static IEnumerable GetEntries(Stream stream, ReaderOptions options) - { - yield return new GZipEntry( - GZipFilePart.Create(stream, options.ArchiveEncoding, options.Providers), - options - ); - } - - // Async methods moved to GZipEntry.Async.cs } diff --git a/src/SharpCompress/Common/GZip/GZipFilePart.Async.cs b/src/SharpCompress/Common/GZip/GZipFilePart.Async.cs index ae3e118a..e4bd555d 100644 --- a/src/SharpCompress/Common/GZip/GZipFilePart.Async.cs +++ b/src/SharpCompress/Common/GZip/GZipFilePart.Async.cs @@ -13,6 +13,7 @@ namespace SharpCompress.Common.GZip; internal sealed partial class GZipFilePart { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal static async ValueTask CreateAsync( Stream stream, IArchiveEncoding archiveEncoding, diff --git a/src/SharpCompress/Common/GZip/GZipFilePart.cs b/src/SharpCompress/Common/GZip/GZipFilePart.cs index 13a1c968..3ce51163 100644 --- a/src/SharpCompress/Common/GZip/GZipFilePart.cs +++ b/src/SharpCompress/Common/GZip/GZipFilePart.cs @@ -16,32 +16,6 @@ internal sealed partial class GZipFilePart : FilePart private readonly Stream _stream; private readonly CompressionProviderRegistry _compressionProviders; - internal static GZipFilePart Create( - Stream stream, - IArchiveEncoding archiveEncoding, - CompressionProviderRegistry compressionProviders - ) - { - var part = new GZipFilePart(stream, archiveEncoding, compressionProviders); - - part.ReadAndValidateGzipHeader(); - if (stream.CanSeek) - { - var position = stream.Position; - stream.Position = stream.Length - 8; - part.ReadTrailer(); - stream.Position = position; - part.EntryStartPosition = position; - } - else - { - // For non-seekable streams, we can't read the trailer or track position. - // Set to 0 since the stream will be read sequentially from its current position. - part.EntryStartPosition = 0; - } - return part; - } - private GZipFilePart( Stream stream, IArchiveEncoding archiveEncoding, @@ -74,6 +48,7 @@ internal sealed partial class GZipFilePart : FilePart private void ReadTrailer() { + // Keep this sync implementation for its stack allocation on the parser hot path. // Read and potentially verify the GZIP trailer: CRC32 and size mod 2^32 Span trailer = stackalloc byte[8]; _stream.ReadFully(trailer); @@ -84,6 +59,7 @@ internal sealed partial class GZipFilePart : FilePart private void ReadAndValidateGzipHeader() { + // Keep this sync implementation for stackalloc and ReadByte-based header parsing. // read the header on the first read Span header = stackalloc byte[10]; var n = _stream.Read(header); @@ -136,6 +112,7 @@ internal sealed partial class GZipFilePart : FilePart private string ReadZeroTerminatedString(Stream stream) { + // Keep this sync implementation for its one-byte stack allocation. Span buf1 = stackalloc byte[1]; var list = new List(); var done = false; diff --git a/src/SharpCompress/Common/IEntryExtensions.Async.cs b/src/SharpCompress/Common/IEntryExtensions.Async.cs index 4ef697e0..eafddfcf 100644 --- a/src/SharpCompress/Common/IEntryExtensions.Async.cs +++ b/src/SharpCompress/Common/IEntryExtensions.Async.cs @@ -38,6 +38,11 @@ internal static partial class IEntryExtensions CancellationToken cancellationToken = default ) { + if (entry.LinkTarget is not null && options.SymbolicLinkHandler is null) + { + return; + } + var destinationFileName = GetEntryDestinationFileName( entry, fullDestinationDirectoryPath, @@ -54,6 +59,20 @@ internal static partial class IEntryExtensions DirectoryManagement.WriteFileOutsideDestinationMessage ); + DirectoryManagement.EnsureNoReparsePointInDestinationDirectory( + destinationFileName, + fullDestinationDirectoryPath + ); + + if (entry.LinkTarget is not null) + { + DirectoryManagement.EnsureLinkTargetInDestinationDirectory( + destinationFileName, + entry.LinkTarget, + fullDestinationDirectoryPath + ); + } + if (writeAsync != null) { await writeAsync(destinationFileName, cancellationToken).ConfigureAwait(false); @@ -69,10 +88,10 @@ internal static partial class IEntryExtensions DirectoryManagement.CreateDirectoryOutsideDestinationMessage ); - if (!Directory.Exists(destinationFileName)) - { - Directory.CreateDirectory(destinationFileName); - } + DirectoryManagement.CreateDirectory( + destinationFileName, + fullDestinationDirectoryPath + ); } } @@ -90,6 +109,8 @@ internal static partial class IEntryExtensions } else { + DirectoryManagement.EnsurePathIsNotReparsePoint(destinationFileName); + var fm = FileMode.Create; if (!options.Overwrite) diff --git a/src/SharpCompress/Common/IEntryExtensions.cs b/src/SharpCompress/Common/IEntryExtensions.cs index 492ad85d..d7e6f5cb 100644 --- a/src/SharpCompress/Common/IEntryExtensions.cs +++ b/src/SharpCompress/Common/IEntryExtensions.cs @@ -45,6 +45,11 @@ internal static partial class IEntryExtensions Action? write ) { + if (entry.LinkTarget is not null && options.SymbolicLinkHandler is null) + { + return; + } + var destinationFileName = GetEntryDestinationFileName( entry, fullDestinationDirectoryPath, @@ -60,6 +65,21 @@ internal static partial class IEntryExtensions fullDestinationDirectoryPath, DirectoryManagement.WriteFileOutsideDestinationMessage ); + + DirectoryManagement.EnsureNoReparsePointInDestinationDirectory( + destinationFileName, + fullDestinationDirectoryPath + ); + + if (entry.LinkTarget is not null) + { + DirectoryManagement.EnsureLinkTargetInDestinationDirectory( + destinationFileName, + entry.LinkTarget, + fullDestinationDirectoryPath + ); + } + write?.Invoke(destinationFileName); } else if (options.ExtractFullPath) @@ -72,10 +92,10 @@ internal static partial class IEntryExtensions DirectoryManagement.CreateDirectoryOutsideDestinationMessage ); - if (!Directory.Exists(destinationFileName)) - { - Directory.CreateDirectory(destinationFileName); - } + DirectoryManagement.CreateDirectory( + destinationFileName, + fullDestinationDirectoryPath + ); } } @@ -102,10 +122,7 @@ internal static partial class IEntryExtensions : DirectoryManagement.WriteFileOutsideDestinationMessage ); - if (!Directory.Exists(destdir)) - { - Directory.CreateDirectory(destdir); - } + DirectoryManagement.CreateDirectory(destdir, fullDestinationDirectoryPath); return Path.Combine(destdir, file); } @@ -126,6 +143,8 @@ internal static partial class IEntryExtensions } else { + DirectoryManagement.EnsurePathIsNotReparsePoint(destinationFileName); + var fm = FileMode.Create; if (!options.Overwrite) diff --git a/src/SharpCompress/Common/Options/IExtractionOptions.cs b/src/SharpCompress/Common/Options/IExtractionOptions.cs index 767bfb93..a6379f91 100644 --- a/src/SharpCompress/Common/Options/IExtractionOptions.cs +++ b/src/SharpCompress/Common/Options/IExtractionOptions.cs @@ -39,6 +39,8 @@ public interface IExtractionOptions /// Delegate for writing symbolic links to disk. /// The first parameter is the source path (where the symlink is created). /// The second parameter is the target path (what the symlink refers to). + /// Directory extraction rejects link targets outside the destination directory and does not + /// follow symbolic links or reparse points in later entry paths. /// Action? SymbolicLinkHandler { get; set; } } diff --git a/src/SharpCompress/Common/Options/IParallelismOptions.cs b/src/SharpCompress/Common/Options/IParallelismOptions.cs new file mode 100644 index 00000000..c398cdf4 --- /dev/null +++ b/src/SharpCompress/Common/Options/IParallelismOptions.cs @@ -0,0 +1,14 @@ +namespace SharpCompress.Common.Options; + +/// +/// Options for controlling whether SharpCompress may use multi-threaded processing. +/// +public interface IParallelismOptions +{ + /// + /// When true, opts in to any optional parallel (multi-threaded) processing a compressor may + /// offer. Compressors that do not implement parallel processing ignore this setting. Default + /// is false (single-threaded processing), so behavior is unchanged unless explicitly enabled. + /// + bool EnableParallelism { get; set; } +} diff --git a/src/SharpCompress/Common/Options/IReaderOptions.cs b/src/SharpCompress/Common/Options/IReaderOptions.cs index c25f4825..71561bfe 100644 --- a/src/SharpCompress/Common/Options/IReaderOptions.cs +++ b/src/SharpCompress/Common/Options/IReaderOptions.cs @@ -3,7 +3,11 @@ using SharpCompress.Providers; namespace SharpCompress.Common.Options; -public interface IReaderOptions : IStreamOptions, IEncodingOptions, IProgressOptions +public interface IReaderOptions + : IStreamOptions, + IEncodingOptions, + IProgressOptions, + IParallelismOptions { /// /// Look for RarArchive (Check for self-extracting archives or cases where RarArchive isn't at the start of the file) diff --git a/src/SharpCompress/Common/Options/IWriterOptions.cs b/src/SharpCompress/Common/Options/IWriterOptions.cs index 37fbb21e..90852c53 100644 --- a/src/SharpCompress/Common/Options/IWriterOptions.cs +++ b/src/SharpCompress/Common/Options/IWriterOptions.cs @@ -7,7 +7,11 @@ namespace SharpCompress.Common.Options; /// /// Options for configuring writer behavior when creating archives. /// -public interface IWriterOptions : IStreamOptions, IEncodingOptions, IProgressOptions +public interface IWriterOptions + : IStreamOptions, + IEncodingOptions, + IProgressOptions, + IParallelismOptions { /// /// The compression type to use for the archive. diff --git a/src/SharpCompress/Common/Rar/RarVolume.cs b/src/SharpCompress/Common/Rar/RarVolume.cs index 1b7de043..f3913d2c 100644 --- a/src/SharpCompress/Common/Rar/RarVolume.cs +++ b/src/SharpCompress/Common/Rar/RarVolume.cs @@ -25,6 +25,8 @@ public abstract class RarVolume : Volume private ArchiveHeader? ArchiveHeader { get; set; } + internal bool IsHeaderEncrypted => _headerFactory.IsEncrypted; + private StreamingMode Mode => _headerFactory.StreamingMode; internal abstract IEnumerable ReadFileParts(); diff --git a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.Async.cs b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.Async.cs index ae2ff232..52bdad89 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.Async.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.Async.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -35,4 +36,71 @@ internal sealed partial class ArchiveDatabase ) .ConfigureAwait(false); } + + /// + /// Async counterpart of the caching GetFolderStream overload. Reuses a cached decoder + /// stream for the folder when possible instead of recreating and re-decoding from the start. + /// + internal async ValueTask GetFolderStreamAsync( + Stream stream, + CFolder folder, + IPasswordProvider pw, + long skipSize, + long entrySize, + CancellationToken cancellationToken + ) + { + if (_cachedFolder == folder && _cachedFolderStream != null) + { + if (skipSize >= _cachedFolderStreamPosition) + { + var delta = skipSize - _cachedFolderStreamPosition; + if (delta > 0) + { + await _cachedFolderStream + .SkipAsync(delta, cancellationToken) + .ConfigureAwait(false); + } + // Assume the caller will fully consume the returned entry stream, advancing the + // shared stream by entrySize bytes. + _cachedFolderStreamPosition = skipSize + entrySize; + return _cachedFolderStream; + } + + // Non-sequential (backward) access within the same folder requires restarting. + await DisposeCachedFolderStreamAsync().ConfigureAwait(false); + } + else if (_cachedFolderStream != null) + { + await DisposeCachedFolderStreamAsync().ConfigureAwait(false); + } + + var newStream = await GetFolderStreamAsync(stream, folder, pw, cancellationToken) + .ConfigureAwait(false); + if (skipSize > 0) + { + await newStream.SkipAsync(skipSize, cancellationToken).ConfigureAwait(false); + } + + _cachedFolder = folder; + _cachedFolderStream = newStream; + _cachedFolderStreamPosition = skipSize + entrySize; + return newStream; + } + + private async ValueTask DisposeCachedFolderStreamAsync() + { + if (_cachedFolderStream is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync().ConfigureAwait(false); + } + else + { +#pragma warning disable VSTHRD103 // Fallback for streams that do not support async disposal. + _cachedFolderStream?.Dispose(); +#pragma warning restore VSTHRD103 + } + _cachedFolderStream = null; + _cachedFolder = null; + } } diff --git a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs index dbad5353..612a0b15 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Linq; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.LZMA.Utilities; +using SharpCompress.IO; namespace SharpCompress.Common.SevenZip; @@ -144,8 +147,30 @@ internal partial class ArchiveDatabase return size; } - internal Stream GetFolderStream(Stream stream, CFolder folder, IPasswordProvider pw) + internal Stream GetFolderStream( + Stream stream, + CFolder folder, + IPasswordProvider pw, + bool enableParallelism = false + ) { +#if !LEGACY_DOTNET + // Opportunistically decode the whole folder concurrently when it's a single, unchained, + // unencrypted LZMA2 stream backed by a real (seekable, positional-I/O-capable) file -- the + // only shape where independent per-block decoding is possible and worth the temp-file + // round-trip. Opt-in only (via ReaderOptions.EnableParallelism): any other shape + // (multi-coder chains, BCJ/delta filters, encryption, a non-file stream, a single-core + // machine, or too few independent restart points to make parallelizing worthwhile) safely + // falls back to the existing sequential decode below. + if ( + enableParallelism + && TryGetParallelDecodedFolderStream(stream, folder, out var parallelStream) + ) + { + return parallelStream!; + } +#endif + var packStreamIndex = folder._firstPackStreamId; var folderStartPackPos = GetFolderStreamPos(folder, 0); var count = folder._packStreams.Count; @@ -163,4 +188,202 @@ internal partial class ArchiveDatabase pw ); } + +#if !LEGACY_DOTNET + /// + /// Attempts to decode entirely up front using + /// , writing the result to a temp file and returning a + /// stream over it. Returns false (and leaves null) whenever + /// the folder's shape, the host stream, or the current hardware make parallel decoding + /// inapplicable or not worthwhile -- callers must fall back to sequential decoding. + /// + private bool TryGetParallelDecodedFolderStream( + Stream stream, + CFolder folder, + out Stream? decodedStream + ) + { + decodedStream = null; + + // Matches 7-Zip's own behavior: with a single available core there is nothing to gain + // from splitting work across threads, so skip straight to the sequential path. + if (Environment.ProcessorCount <= 1) + { + return false; + } + + // Only a lone, unchained LZMA2 coder can be split into independently-decodable blocks. + // Any bind pairs (BCJ/delta filters, etc.) or additional coders (e.g. AES encryption) + // mean the folder's bytes depend on more than just this one LZMA2 stream. + if ( + folder._coders.Count != 1 + || folder._coders[0]._methodId != CMethodId.K_LZMA2 + || folder._packStreams.Count != 1 + || folder._bindPairs.Count != 0 + ) + { + return false; + } + + var props = folder._coders[0]._props; + if (props is null || props.Length == 0) + { + return false; + } + + // A file handle is not sufficient by itself: packStart is an offset in this stream's + // logical byte sequence. Only use positional I/O when that sequence is exactly one file. + // In particular, SourceStream concatenates multipart archives, where a logical offset + // cannot be passed directly to a handle for just one physical part. + if (!TryGetPositionallyEquivalentInputFile(stream, out var inputFile)) + { + return false; + } + + var folderIndex = _folders.IndexOf(folder); + var packStart = GetFolderStreamPos(folder, 0); + var packSize = GetFolderFullPackSize(folderIndex); + var unpackSize = folder.GetUnpackSize(); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(stream, packStart, packSize, unpackSize); + if (blocks is null || blocks.Count <= 1) + { + return false; + } + + var tempFile = new FileStream( + Path.GetTempFileName(), + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + 4096, + FileOptions.DeleteOnClose + ); + + try + { + tempFile.SetLength(unpackSize); + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + props, + packStart, + blocks, + tempFile.SafeFileHandle, + Math.Min(Environment.ProcessorCount, Lzma2ParallelDecoder.MaxThreads) + ); + } + catch + { + tempFile.Dispose(); + throw; + } + + tempFile.Position = 0; + decodedStream = tempFile; + return true; + } + + private static bool TryGetPositionallyEquivalentInputFile( + Stream stream, + [NotNullWhen(true)] out FileStream? inputFile + ) + { + inputFile = null; + + if (stream is FileStream fileStream) + { + inputFile = fileStream; + return true; + } + + // Volume wraps caller-owned streams in this non-disposing passthrough wrapper. It does + // not change positions, so its single SourceStream input can still be used safely. + if (stream is SharpCompressStream { IsPassthrough: true } passthrough) + { + stream = passthrough.BaseStream(); + } + + if (stream is not SourceStream sourceStream) + { + return false; + } + + var streams = sourceStream.Streams.Take(2).ToArray(); + if (streams.Length != 1 || streams[0] is not FileStream sourceFile) + { + return false; + } + + inputFile = sourceFile; + return true; + } +#endif + + // Cache used to avoid re-decoding a solid folder from scratch for every file it contains. + // Without this, extracting N files from the same solid folder decodes O(N^2) bytes since each + // file's stream was previously created fresh from the folder start and skipped forward. + private CFolder? _cachedFolder; + private Stream? _cachedFolderStream; + private long _cachedFolderStreamPosition; + + /// + /// Returns a stream positioned at bytes into the decompressed + /// contents of . Reuses a cached decoder stream for the folder when + /// possible instead of recreating and re-decoding from the start. + /// + internal Stream GetFolderStream( + Stream stream, + CFolder folder, + IPasswordProvider pw, + long skipSize, + long entrySize, + bool enableParallelism = false + ) + { + if (_cachedFolder == folder && _cachedFolderStream != null) + { + if (skipSize >= _cachedFolderStreamPosition) + { + var delta = skipSize - _cachedFolderStreamPosition; + if (delta > 0) + { + _cachedFolderStream.Skip(delta); + } + // Assume the caller will fully consume the returned entry stream, advancing the + // shared stream by entrySize bytes. + _cachedFolderStreamPosition = skipSize + entrySize; + return _cachedFolderStream; + } + + // Non-sequential (backward) access within the same folder requires restarting. + _cachedFolderStream.Dispose(); + _cachedFolderStream = null; + _cachedFolder = null; + } + else if (_cachedFolderStream != null) + { + _cachedFolderStream.Dispose(); + _cachedFolderStream = null; + _cachedFolder = null; + } + + var newStream = GetFolderStream(stream, folder, pw, enableParallelism); + if (skipSize > 0) + { + newStream.Skip(skipSize); + } + + _cachedFolder = folder; + _cachedFolderStream = newStream; + _cachedFolderStreamPosition = skipSize + entrySize; + return newStream; + } + + internal void DisposeCachedFolderStream() + { + _cachedFolderStream?.Dispose(); + _cachedFolderStream = null; + _cachedFolder = null; + _cachedFolderStreamPosition = 0; + } } diff --git a/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs b/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs index 52964077..45420797 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs @@ -211,7 +211,11 @@ internal sealed partial class ArchiveReader await outStream .ReadExactAsync(data, 0, data.Length, cancellationToken) .ConfigureAwait(false); - if (outStream.ReadByte() >= 0) + if ( + await outStream + .ReadAsync(new byte[1], 0, 1, cancellationToken) + .ConfigureAwait(false) > 0 + ) { throw new InvalidFormatException("Decoded stream is longer than expected."); } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipFilePart.cs b/src/SharpCompress/Common/SevenZip/SevenZipFilePart.cs index 65bb7503..007b3508 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipFilePart.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipFilePart.cs @@ -11,18 +11,21 @@ internal class SevenZipFilePart : FilePart private CompressionType? _type; private readonly Stream _stream; private readonly ArchiveDatabase _database; + private readonly bool _enableParallelism; internal SevenZipFilePart( Stream stream, ArchiveDatabase database, int index, CFileItem fileEntry, - IArchiveEncoding archiveEncoding + IArchiveEncoding archiveEncoding, + bool enableParallelism = false ) : base(archiveEncoding) { _stream = stream; _database = database; + _enableParallelism = enableParallelism; Index = index; Header = fileEntry; if (Header.HasStream) @@ -44,7 +47,6 @@ internal class SevenZipFilePart : FilePart { return Stream.Null; } - var folderStream = _database.GetFolderStream(_stream, Folder!, _database.PasswordProvider); var firstFileIndex = _database._folderStartFileIndex[_database._folders.IndexOf(Folder!)]; var skipCount = Index - firstFileIndex; @@ -53,11 +55,16 @@ internal class SevenZipFilePart : FilePart { skipSize += _database._files[firstFileIndex + i].Size; } - if (skipSize > 0) - { - folderStream.Skip(skipSize); - } - return new ReadOnlySubStream(folderStream, Header.Size, leaveOpen: false); + + var folderStream = _database.GetFolderStream( + _stream, + Folder!, + _database.PasswordProvider, + skipSize, + Header.Size, + _enableParallelism + ); + return new ReadOnlySubStream(folderStream, Header.Size, leaveOpen: true); } internal override async ValueTask GetCompressedStreamAsync( @@ -68,9 +75,6 @@ internal class SevenZipFilePart : FilePart { return Stream.Null; } - var folderStream = await _database - .GetFolderStreamAsync(_stream, Folder!, _database.PasswordProvider, cancellationToken) - .ConfigureAwait(false); var firstFileIndex = _database._folderStartFileIndex[_database._folders.IndexOf(Folder!)]; var skipCount = Index - firstFileIndex; @@ -79,11 +83,18 @@ internal class SevenZipFilePart : FilePart { skipSize += _database._files[firstFileIndex + i].Size; } - if (skipSize > 0) - { - await folderStream.SkipAsync(skipSize, cancellationToken).ConfigureAwait(false); - } - return new ReadOnlySubStream(folderStream, Header.Size, leaveOpen: false); + + var folderStream = await _database + .GetFolderStreamAsync( + _stream, + Folder!, + _database.PasswordProvider, + skipSize, + Header.Size, + cancellationToken + ) + .ConfigureAwait(false); + return new ReadOnlySubStream(folderStream, Header.Size, leaveOpen: true); } public CompressionType CompressionType diff --git a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs index 86594f63..c4dc0b72 100644 --- a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs +++ b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs @@ -29,18 +29,7 @@ internal class TarReadOnlySubStream : Stream _isDisposed = true; if (disposing) { - if (Utility.UseSyncOverAsyncDispose()) - { -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits -#pragma warning disable CA2012 - AdvanceToNextHeaderAsync().GetAwaiter().GetResult(); -#pragma warning restore CA2012 -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - else - { - AdvanceToNextHeader(); - } + AdvanceToNextHeader(); } base.Dispose(disposing); } diff --git a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.Async.cs b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.Async.cs index f541fc8d..e43af8c7 100644 --- a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.Async.cs @@ -7,6 +7,7 @@ namespace SharpCompress.Common.Zip.Headers; internal abstract partial class ZipFileEntry { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal async ValueTask ComposeEncryptionDataAsync( Stream archiveStream, CancellationToken cancellationToken = default @@ -15,7 +16,7 @@ internal abstract partial class ZipFileEntry ThrowHelper.ThrowIfNull(archiveStream); var buffer = new byte[12]; - await archiveStream.ReadFullyAsync(buffer, 0, 12, cancellationToken).ConfigureAwait(false); + await archiveStream.ReadFullyAsync(buffer, cancellationToken).ConfigureAwait(false); var encryptionData = PkwareTraditionalEncryptionData.ForRead(Password!, this, buffer); diff --git a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs index e7f0b042..68fa65d3 100644 --- a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs +++ b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs @@ -42,18 +42,6 @@ internal abstract partial class ZipFileEntry(ZipHeaderType type, IArchiveEncodin public string? Password { get; set; } - internal PkwareTraditionalEncryptionData ComposeEncryptionData(Stream archiveStream) - { - ThrowHelper.ThrowIfNull(archiveStream); - - var buffer = new byte[12]; - archiveStream.ReadFully(buffer); - - var encryptionData = PkwareTraditionalEncryptionData.ForRead(Password!, this, buffer); - - return encryptionData; - } - internal WinzipAesEncryptionData? WinzipAesEncryptionData { get; set; } /// diff --git a/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.Async.cs b/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.Async.cs index 32a0a7bf..63abcf4d 100644 --- a/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.Async.cs +++ b/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.Async.cs @@ -8,6 +8,7 @@ namespace SharpCompress.Common.Zip; internal partial class PkwareTraditionalCryptoStream { + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, @@ -59,6 +60,7 @@ internal partial class PkwareTraditionalCryptoStream } #endif + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.cs b/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.cs index 2efd036e..9545d8de 100644 --- a/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.cs +++ b/src/SharpCompress/Common/Zip/PkwareTraditionalCryptoStream.cs @@ -41,49 +41,6 @@ internal partial class PkwareTraditionalCryptoStream : Stream set => throw new NotSupportedException(); } - public override int Read(byte[] buffer, int offset, int count) - { - if (_mode == CryptoMode.Encrypt) - { - throw new NotSupportedException("This stream does not encrypt via Read()"); - } - - ThrowHelper.ThrowIfNull(buffer); - - var temp = new byte[count]; - var readBytes = _stream.Read(temp, 0, count); - var decrypted = _encryptor.Decrypt(temp, readBytes); - Buffer.BlockCopy(decrypted, 0, buffer, offset, readBytes); - return readBytes; - } - - public override void Write(byte[] buffer, int offset, int count) - { - if (_mode == CryptoMode.Decrypt) - { - throw new NotSupportedException("This stream does not Decrypt via Write()"); - } - - if (count == 0) - { - return; - } - - byte[] plaintext; - if (offset != 0) - { - plaintext = new byte[count]; - Buffer.BlockCopy(buffer, offset, plaintext, 0, count); - } - else - { - plaintext = buffer; - } - - var encrypted = _encryptor.Encrypt(plaintext, count); - _stream.Write(encrypted, 0, encrypted.Length); - } - public override void Flush() { } public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); diff --git a/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs b/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs index 8fff8436..c81c11ed 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipFilePart.Async.cs @@ -7,6 +7,7 @@ namespace SharpCompress.Common.Zip; internal partial class SeekableZipFilePart { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal override async ValueTask GetCompressedStreamAsync( CancellationToken cancellationToken = default ) @@ -19,8 +20,15 @@ internal partial class SeekableZipFilePart return await base.GetCompressedStreamAsync(cancellationToken).ConfigureAwait(false); } + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask LoadLocalHeaderAsync(CancellationToken cancellationToken = default) => Header = await _headerFactory - .GetLocalHeaderAsync(BaseStream, (DirectoryEntryHeader)Header) + .GetLocalHeaderAsync(BaseStream, _directoryEntryHeader) + .ConfigureAwait(false); + + [Zomp.SyncMethodGenerator.CreateSyncVersion] + internal async ValueTask GetRawLocalHeaderAsync() => + await _headerFactory + .GetRawLocalHeaderAsync(BaseStream, _directoryEntryHeader) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs b/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs index f9c59dfd..baab3218 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipFilePart.cs @@ -1,6 +1,5 @@ using System.IO; using SharpCompress.Common.Zip.Headers; -using SharpCompress.Compressors; using SharpCompress.Providers; namespace SharpCompress.Common.Zip; @@ -9,6 +8,7 @@ internal partial class SeekableZipFilePart : ZipFilePart { private bool _isLocalHeaderLoaded; private readonly SeekableZipHeaderFactory _headerFactory; + private readonly DirectoryEntryHeader _directoryEntryHeader; internal SeekableZipFilePart( SeekableZipHeaderFactory headerFactory, @@ -16,20 +16,14 @@ internal partial class SeekableZipFilePart : ZipFilePart Stream stream, CompressionProviderRegistry compressionProviders ) - : base(header, stream, compressionProviders) => _headerFactory = headerFactory; - - internal override Stream GetCompressedStream() + : base(header, stream, compressionProviders) { - if (!_isLocalHeaderLoaded) - { - LoadLocalHeader(); - _isLocalHeaderLoaded = true; - } - return base.GetCompressedStream(); + _headerFactory = headerFactory; + _directoryEntryHeader = header; } - private void LoadLocalHeader() => - Header = _headerFactory.GetLocalHeader(BaseStream, (DirectoryEntryHeader)Header); + internal bool HasDeferredSizes => + FlagUtility.HasFlag(_directoryEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor); protected override Stream CreateBaseStream() { diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs index 11e7eec4..e9065f80 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.Async.cs @@ -146,17 +146,27 @@ internal sealed partial class SeekableZipHeaderFactory throw new ArchiveOperationException(); } - // populate fields only known from the DirectoryEntryHeader - localEntryHeader.HasData = directoryEntryHeader.HasData; - localEntryHeader.ExternalFileAttributes = directoryEntryHeader.ExternalFileAttributes; - localEntryHeader.Comment = directoryEntryHeader.Comment; - - if (FlagUtility.HasFlag(localEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor)) - { - localEntryHeader.Crc = directoryEntryHeader.Crc; - localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; - localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; - } + PopulateDirectoryEntryMetadata(localEntryHeader, directoryEntryHeader); return localEntryHeader; } + + internal async ValueTask GetRawLocalHeaderAsync( + Stream stream, + DirectoryEntryHeader directoryEntryHeader + ) + { + stream.Seek(directoryEntryHeader.RelativeOffsetOfEntryHeader, SeekOrigin.Begin); +#if NET8_0_OR_GREATER + await using var reader = new AsyncBinaryReader(stream, leaveOpen: true); +#else + using var reader = new AsyncBinaryReader(stream, leaveOpen: true); +#endif + var signature = await reader.ReadUInt32Async().ConfigureAwait(false); + if (signature != ENTRY_HEADER_BYTES) + { + throw new ArchiveOperationException(); + } + + return await ReadLocalHeader(reader).ConfigureAwait(false); + } } diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs index 7a88a480..c20163b9 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs @@ -151,6 +151,31 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory throw new ArchiveOperationException(); } + PopulateDirectoryEntryMetadata(localEntryHeader, directoryEntryHeader); + return localEntryHeader; + } + + internal LocalEntryHeader GetRawLocalHeader( + Stream stream, + DirectoryEntryHeader directoryEntryHeader + ) + { + stream.Seek(directoryEntryHeader.RelativeOffsetOfEntryHeader, SeekOrigin.Begin); + var reader = new BinaryReader(stream); + var signature = reader.ReadUInt32(); + if (signature != ENTRY_HEADER_BYTES) + { + throw new ArchiveOperationException(); + } + + return ReadLocalHeader(reader); + } + + private static void PopulateDirectoryEntryMetadata( + LocalEntryHeader localEntryHeader, + DirectoryEntryHeader directoryEntryHeader + ) + { // populate fields only known from the DirectoryEntryHeader localEntryHeader.HasData = directoryEntryHeader.HasData; localEntryHeader.ExternalFileAttributes = directoryEntryHeader.ExternalFileAttributes; @@ -163,6 +188,5 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; } - return localEntryHeader; } } diff --git a/src/SharpCompress/Common/Zip/StreamingZipFilePart.Async.cs b/src/SharpCompress/Common/Zip/StreamingZipFilePart.Async.cs index dbd62841..fa372826 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipFilePart.Async.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipFilePart.Async.cs @@ -7,6 +7,7 @@ namespace SharpCompress.Common.Zip; internal sealed partial class StreamingZipFilePart { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal override async ValueTask GetCompressedStreamAsync( CancellationToken cancellationToken = default ) @@ -15,17 +16,18 @@ internal sealed partial class StreamingZipFilePart { return Stream.Null; } - _decompressionStream = await CreateDecompressionStreamAsync( + var decompressionStream = await CreateDecompressionStreamAsync( await GetCryptoStreamAsync(CreateBaseStream(), cancellationToken) .ConfigureAwait(false), Header.CompressionMethod, cancellationToken ) .ConfigureAwait(false); + _decompressionStream = decompressionStream; if (LeaveStreamOpen) { - return SharpCompressStream.CreateNonDisposing(_decompressionStream); + return SharpCompressStream.CreateNonDisposing(decompressionStream); } - return _decompressionStream; + return decompressionStream; } } diff --git a/src/SharpCompress/Common/Zip/StreamingZipFilePart.cs b/src/SharpCompress/Common/Zip/StreamingZipFilePart.cs index e8ef7c55..9240e16c 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipFilePart.cs @@ -1,6 +1,5 @@ using System.IO; using SharpCompress.Common.Zip.Headers; -using SharpCompress.Compressors; using SharpCompress.IO; using SharpCompress.Providers; @@ -19,23 +18,6 @@ internal sealed partial class StreamingZipFilePart : ZipFilePart protected override Stream CreateBaseStream() => Header.PackedStream.NotNull(); - internal override Stream GetCompressedStream() - { - if (!Header.HasData) - { - return Stream.Null; - } - _decompressionStream = CreateDecompressionStream( - GetCryptoStream(CreateBaseStream()), - Header.CompressionMethod - ); - if (LeaveStreamOpen) - { - return SharpCompressStream.CreateNonDisposing(_decompressionStream); - } - return _decompressionStream; - } - internal BinaryReader FixStreamedFileLocation(ref Stream stream) { if (Header.IsDirectory) @@ -45,12 +27,12 @@ internal sealed partial class StreamingZipFilePart : ZipFilePart if (Header.HasData && !Skipped) { - _decompressionStream ??= GetCompressedStream(); + var decompressionStream = _decompressionStream ??= GetCompressedStream().NotNull(); - _decompressionStream.Skip(); + decompressionStream.Skip(); // If we had TotalIn / TotalOut we could have used them - Header.CompressedSize = _decompressionStream.Position; + Header.CompressedSize = decompressionStream.Position; Skipped = true; } diff --git a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.Async.cs b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.Async.cs index d68b8a87..04863349 100644 --- a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.Async.cs +++ b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.Async.cs @@ -32,6 +32,7 @@ internal partial class WinzipAesCryptoStream } #endif + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs index 9d269473..8a4eda18 100644 --- a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs +++ b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs @@ -69,28 +69,27 @@ internal partial class WinzipAesCryptoStream : Stream _isDisposed = true; if (disposing) { - // Read out last 10 auth bytes - catch exceptions for async-only streams - if (Utility.UseSyncOverAsyncDispose()) + // Read out last 10 auth bytes +#if LEGACY_DOTNET + // Stream has no DisposeAsync on legacy targets, so async flows fall back to this + // sync Dispose while the underlying stream may be async-only. + var ten = ArrayPool.Shared.Rent(10); + try { - var ten = ArrayPool.Shared.Rent(10); - try - { #pragma warning disable VSTHRD002 // Avoid problematic synchronous waits #pragma warning disable CA2012 - _stream.ReadFullyAsync(ten, 0, 10).GetAwaiter().GetResult(); + _stream.ReadFullyAsync(ten, 0, 10).GetAwaiter().GetResult(); #pragma warning restore CA2012 #pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - finally - { - ArrayPool.Shared.Return(ten); - } } - else + finally { - Span ten = stackalloc byte[10]; - _stream.ReadFully(ten); + ArrayPool.Shared.Return(ten); } +#else + Span ten = stackalloc byte[10]; + _stream.ReadFully(ten); +#endif _stream.Dispose(); } base.Dispose(disposing); @@ -104,25 +103,6 @@ internal partial class WinzipAesCryptoStream : Stream public override void Flush() { } - public override int Read(byte[] buffer, int offset, int count) - { - if (_totalBytesLeftToRead == 0) - { - return 0; - } - var bytesToRead = count; - if (count > _totalBytesLeftToRead) - { - bytesToRead = (int)_totalBytesLeftToRead; - } - var read = _stream.Read(buffer, offset, bytesToRead); - _totalBytesLeftToRead -= read; - - ReadTransformBlocks(buffer, offset, read); - - return read; - } - private void FillCounterOut() { // update the counter diff --git a/src/SharpCompress/Common/Zip/ZipFilePart.Async.cs b/src/SharpCompress/Common/Zip/ZipFilePart.Async.cs index 29f523a6..2902de0b 100644 --- a/src/SharpCompress/Common/Zip/ZipFilePart.Async.cs +++ b/src/SharpCompress/Common/Zip/ZipFilePart.Async.cs @@ -13,6 +13,7 @@ namespace SharpCompress.Common.Zip; internal abstract partial class ZipFilePart { + [Zomp.SyncMethodGenerator.CreateSyncVersion] internal override async ValueTask GetCompressedStreamAsync( CancellationToken cancellationToken = default ) @@ -35,6 +36,7 @@ internal abstract partial class ZipFilePart return decompressionStream; } + [Zomp.SyncMethodGenerator.CreateSyncVersion] protected async ValueTask GetCryptoStreamAsync( Stream plainStream, CancellationToken cancellationToken = default @@ -242,6 +244,7 @@ internal abstract partial class ZipFilePart } } + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask CreateWinzipAesDecompressionStreamAsync( Stream stream, CancellationToken cancellationToken = default diff --git a/src/SharpCompress/Common/Zip/ZipFilePart.cs b/src/SharpCompress/Common/Zip/ZipFilePart.cs index de182f37..8597b825 100644 --- a/src/SharpCompress/Common/Zip/ZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/ZipFilePart.cs @@ -1,19 +1,6 @@ using System; -using System.Buffers.Binary; using System.IO; -using System.Linq; using SharpCompress.Common.Zip.Headers; -using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; -using SharpCompress.Compressors.Deflate; -using SharpCompress.Compressors.Deflate64; -using SharpCompress.Compressors.Explode; -using SharpCompress.Compressors.LZMA; -using SharpCompress.Compressors.PPMd; -using SharpCompress.Compressors.Reduce; -using SharpCompress.Compressors.Shrink; -using SharpCompress.Compressors.Xz; -using SharpCompress.Compressors.ZStandard; using SharpCompress.IO; using SharpCompress.Providers; @@ -41,23 +28,6 @@ internal abstract partial class ZipFilePart : FilePart internal override string? FilePartName => Header.Name; - internal override Stream GetCompressedStream() - { - if (!Header.HasData) - { - return Stream.Null; - } - var decompressionStream = CreateDecompressionStream( - GetCryptoStream(CreateBaseStream()), - Header.CompressionMethod - ); - if (LeaveStreamOpen) - { - return SharpCompressStream.CreateNonDisposing(decompressionStream); - } - return decompressionStream; - } - internal override Stream GetRawStream() { if (!Header.HasData) @@ -192,101 +162,4 @@ internal abstract partial class ZipFilePart : FilePart // For simple methods, use the basic decompress return providers.CreateDecompressStream(compressionType, stream, context); } - - private Stream CreateWinzipAesDecompressionStream(Stream stream) - { - var data = Header.Extra.SingleOrDefault(x => x.Type == ExtraDataType.WinZipAes); - if (data is null) - { - throw new InvalidFormatException("No Winzip AES extra data found."); - } - if (data.Length != 7) - { - throw new InvalidFormatException("Winzip data length is not 7."); - } - var compressedMethod = BinaryPrimitives.ReadUInt16LittleEndian(data.DataBytes); - - if (compressedMethod != 0x01 && compressedMethod != 0x02) - { - throw new InvalidFormatException( - "Unexpected vendor version number for WinZip AES metadata" - ); - } - - var vendorId = BinaryPrimitives.ReadUInt16LittleEndian(data.DataBytes.AsSpan(2)); - if (vendorId != 0x4541) - { - throw new InvalidFormatException("Unexpected vendor ID for WinZip AES metadata"); - } - return CreateDecompressionStream( - stream, - (ZipCompressionMethod)BinaryPrimitives.ReadUInt16LittleEndian(data.DataBytes.AsSpan(5)) - ); - } - - protected Stream GetCryptoStream(Stream plainStream) - { - var isFileEncrypted = FlagUtility.HasFlag(Header.Flags, HeaderFlags.Encrypted); - - if (Header.CompressedSize == 0 && isFileEncrypted) - { - throw new NotSupportedException("Cannot encrypt file with unknown size at start."); - } - - if ( - Header.CompressedSize == 0 - && FlagUtility.HasFlag(Header.Flags, HeaderFlags.UsePostDataDescriptor) - ) - { - plainStream = SharpCompressStream.CreateNonDisposing(plainStream); //make sure AES doesn't close - } - else - { - plainStream = new ReadOnlySubStream(plainStream, Header.CompressedSize); //make sure AES doesn't close - } - - if (isFileEncrypted) - { - switch (Header.CompressionMethod) - { - case ZipCompressionMethod.None: - case ZipCompressionMethod.Shrink: - case ZipCompressionMethod.Reduce1: - case ZipCompressionMethod.Reduce2: - case ZipCompressionMethod.Reduce3: - case ZipCompressionMethod.Reduce4: - case ZipCompressionMethod.Deflate: - case ZipCompressionMethod.Deflate64: - case ZipCompressionMethod.BZip2: - case ZipCompressionMethod.LZMA: - case ZipCompressionMethod.PPMd: - { - return new PkwareTraditionalCryptoStream( - plainStream, - Header.ComposeEncryptionData(plainStream), - CryptoMode.Decrypt - ); - } - - case ZipCompressionMethod.WinzipAes: - { - if (Header.WinzipAesEncryptionData != null) - { - return new WinzipAesCryptoStream( - plainStream, - Header.WinzipAesEncryptionData, - Header.CompressedSize - 10 - ); - } - return plainStream; - } - - default: - { - throw new ArchiveOperationException("Header.CompressionMethod is invalid"); - } - } - } - return plainStream; - } } diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs index bc5944b4..bb420a59 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs @@ -20,8 +20,7 @@ internal partial class ZipHeaderFactory { case ENTRY_HEADER_BYTES: { - var entryHeader = new LocalEntryHeader(_archiveEncoding); - await entryHeader.Read(reader).ConfigureAwait(false); + var entryHeader = await ReadLocalHeader(reader).ConfigureAwait(false); await LoadHeaderAsync(entryHeader, reader.BaseStream).ConfigureAwait(false); _lastEntryHeader = entryHeader; @@ -87,6 +86,13 @@ internal partial class ZipHeaderFactory } } + protected async ValueTask ReadLocalHeader(AsyncBinaryReader reader) + { + var entryHeader = new LocalEntryHeader(_archiveEncoding); + await entryHeader.Read(reader).ConfigureAwait(false); + return entryHeader; + } + /// /// Loads encryption metadata and stream positioning for a header using async reads where needed. /// diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs index e1ebb8c1..f98620f9 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs @@ -42,8 +42,7 @@ internal partial class ZipHeaderFactory { case ENTRY_HEADER_BYTES: { - var entryHeader = new LocalEntryHeader(_archiveEncoding); - entryHeader.Read(reader); + var entryHeader = ReadLocalHeader(reader); LoadHeader(entryHeader, reader.BaseStream); _lastEntryHeader = entryHeader; @@ -109,6 +108,13 @@ internal partial class ZipHeaderFactory } } + protected LocalEntryHeader ReadLocalHeader(BinaryReader reader) + { + var entryHeader = new LocalEntryHeader(_archiveEncoding); + entryHeader.Read(reader); + return entryHeader; + } + internal static bool IsHeader(uint headerBytes) { switch (headerBytes) diff --git a/src/SharpCompress/Compressors/Deflate/DeflateStream.Async.cs b/src/SharpCompress/Compressors/Deflate/DeflateStream.Async.cs index 454ba022..ae6343cd 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateStream.Async.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateStream.Async.cs @@ -29,6 +29,10 @@ public partial class DeflateStream #endif } + /// + /// Flush the stream. + /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task FlushAsync(CancellationToken cancellationToken) { if (_disposed) @@ -38,6 +42,24 @@ public partial class DeflateStream await _baseStream.FlushAsync(cancellationToken).ConfigureAwait(false); } + /// + /// Read data from the stream. + /// + /// + /// + /// If you wish to use the DeflateStream to compress data while reading, create it with + /// CompressionMode.Compress and an uncompressed data stream. Reading then returns + /// compressed data. With CompressionMode.Decompress, reading returns decompressed data. + /// + /// + /// A DeflateStream can be used for Read() or Write(), but not both. + /// + /// + /// The buffer into which the read data should be placed. + /// The offset within that data array to put the first byte read. + /// The number of bytes to read. + /// The number of bytes actually read. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, @@ -68,6 +90,23 @@ public partial class DeflateStream } #endif + /// + /// Write data to the stream. + /// + /// + /// + /// With CompressionMode.Compress, written uncompressed data is compressed to the + /// destination. With CompressionMode.Decompress, written compressed data is decompressed + /// to the destination. + /// + /// + /// A DeflateStream can be used for Read() or Write(), but not both. + /// + /// + /// The buffer holding data to write to the stream. + /// The offset within that data array to find the first byte to write. + /// The number of bytes to write. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs index 13f3b356..47ab0c7f 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs @@ -27,8 +27,6 @@ using System; using System.IO; using System.Text; -using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.IO; @@ -270,54 +268,6 @@ public partial class DeflateStream : Stream, IStreamStack Stream IStreamStack.BaseStream() => _baseStream; - /// - /// Flush the stream. - /// - public override void Flush() - { - if (_disposed) - { - throw new ObjectDisposedException("DeflateStream"); - } - _baseStream.Flush(); - } - - /// - /// Read data from the stream. - /// - /// - /// - /// - /// If you wish to use the DeflateStream to compress data while - /// reading, you can create a DeflateStream with - /// CompressionMode.Compress, providing an uncompressed data stream. - /// Then call Read() on that DeflateStream, and the data read will be - /// compressed as you read. If you wish to use the DeflateStream to - /// decompress data while reading, you can create a DeflateStream with - /// CompressionMode.Decompress, providing a readable compressed data - /// stream. Then call Read() on that DeflateStream, and the data read - /// will be decompressed as you read. - /// - /// - /// - /// A DeflateStream can be used for Read() or Write(), but not both. - /// - /// - /// - /// The buffer into which the read data should be placed. - /// the offset within that data array to put the first byte read. - /// the number of bytes to read. - /// the number of bytes actually read - public override int Read(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("DeflateStream"); - } - - return _baseStream.Read(buffer, offset, count); - } - public override int ReadByte() { if (_disposed) @@ -341,44 +291,6 @@ public partial class DeflateStream : Stream, IStreamStack /// this is irrelevant, since it will always throw! public override void SetLength(long value) => throw new NotSupportedException(); - /// - /// Write data to the stream. - /// - /// - /// - /// - /// If you wish to use the DeflateStream to compress data while - /// writing, you can create a DeflateStream with - /// CompressionMode.Compress, and a writable output stream. Then call - /// Write() on that DeflateStream, providing uncompressed data - /// as input. The data sent to the output stream will be the compressed form - /// of the data written. If you wish to use the DeflateStream to - /// decompress data while writing, you can create a DeflateStream with - /// CompressionMode.Decompress, and a writable output stream. Then - /// call Write() on that stream, providing previously compressed - /// data. The data sent to the output stream will be the decompressed form of - /// the data written. - /// - /// - /// - /// A DeflateStream can be used for Read() or Write(), - /// but not both. - /// - /// - /// - /// - /// The buffer holding data to write to the stream. - /// the offset within that data array to find the first byte to write. - /// the number of bytes to write. - public override void Write(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("DeflateStream"); - } - _baseStream.Write(buffer, offset, count); - } - public override void WriteByte(byte value) { if (_disposed) diff --git a/src/SharpCompress/Compressors/Deflate/GZipStream.Async.cs b/src/SharpCompress/Compressors/Deflate/GZipStream.Async.cs index dc2f823c..4a70c519 100644 --- a/src/SharpCompress/Compressors/Deflate/GZipStream.Async.cs +++ b/src/SharpCompress/Compressors/Deflate/GZipStream.Async.cs @@ -9,6 +9,10 @@ namespace SharpCompress.Compressors.Deflate; public partial class GZipStream { + /// + /// Flush the stream. + /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task FlushAsync(CancellationToken cancellationToken) { if (_disposed) @@ -18,6 +22,38 @@ public partial class GZipStream await BaseStream.FlushAsync(cancellationToken).ConfigureAwait(false); } + /// + /// Read and decompress data from the source stream. + /// + /// + /// + /// With a GZipStream, decompression is done through reading. + /// + /// + /// + /// + /// byte[] working = new byte[WORKING_BUFFER_SIZE]; + /// using (System.IO.Stream input = System.IO.File.OpenRead(_CompressedFile)) + /// { + /// using (Stream decompressor= new Ionic.Zlib.GZipStream(input, CompressionMode.Decompress, true)) + /// { + /// using (var output = System.IO.File.Create(_DecompressedFile)) + /// { + /// int n; + /// while ((n= decompressor.Read(working, 0, working.Length)) !=0) + /// { + /// output.Write(working, 0, n); + /// } + /// } + /// } + /// } + /// + /// + /// The buffer into which the decompressed data should be placed. + /// the offset within that data array to put the first byte read. + /// the number of bytes to read. + /// the number of bytes actually read + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, @@ -33,6 +69,9 @@ public partial class GZipStream .ReadAsync(buffer, offset, count, cancellationToken) .ConfigureAwait(false); + // Console.WriteLine("GZipStream::Read(buffer, off({0}), c({1}) = {2}", offset, count, n); + // Console.WriteLine( Util.FormatByteArray(buffer, offset, n) ); + if (!_firstReadDone) { _firstReadDone = true; @@ -66,6 +105,29 @@ public partial class GZipStream } #endif + /// + /// Write data to the stream. + /// + /// + /// + /// + /// If you wish to use the GZipStream to compress data while writing, + /// you can create a GZipStream with CompressionMode.Compress, and a + /// writable output stream. Then call Write() on that GZipStream, + /// providing uncompressed data as input. The data sent to the output stream + /// will be the compressed form of the data written. + /// + /// + /// + /// A GZipStream can be used for Read() or Write(), but not + /// both. Writing implies compression. Reading implies decompression. + /// + /// + /// + /// The buffer holding data to write to the stream. + /// the offset within that data array to find the first byte to write. + /// the number of bytes to write. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, @@ -79,6 +141,7 @@ public partial class GZipStream } if (BaseStream._streamMode == ZlibBaseStream.StreamMode.Undefined) { + //Console.WriteLine("GZipStream: First write"); if (BaseStream._wantCompress) { // first write in compression, therefore, emit the GZIP header diff --git a/src/SharpCompress/Compressors/Deflate/GZipStream.cs b/src/SharpCompress/Compressors/Deflate/GZipStream.cs index b0a690d8..0009118d 100644 --- a/src/SharpCompress/Compressors/Deflate/GZipStream.cs +++ b/src/SharpCompress/Compressors/Deflate/GZipStream.cs @@ -250,70 +250,6 @@ public partial class GZipStream : Stream } } - /// - /// Flush the stream. - /// - public override void Flush() - { - if (_disposed) - { - throw new ObjectDisposedException("GZipStream"); - } - BaseStream.Flush(); - } - - /// - /// Read and decompress data from the source stream. - /// - /// - /// - /// With a GZipStream, decompression is done through reading. - /// - /// - /// - /// - /// byte[] working = new byte[WORKING_BUFFER_SIZE]; - /// using (System.IO.Stream input = System.IO.File.OpenRead(_CompressedFile)) - /// { - /// using (Stream decompressor= new Ionic.Zlib.GZipStream(input, CompressionMode.Decompress, true)) - /// { - /// using (var output = System.IO.File.Create(_DecompressedFile)) - /// { - /// int n; - /// while ((n= decompressor.Read(working, 0, working.Length)) !=0) - /// { - /// output.Write(working, 0, n); - /// } - /// } - /// } - /// } - /// - /// - /// The buffer into which the decompressed data should be placed. - /// the offset within that data array to put the first byte read. - /// the number of bytes to read. - /// the number of bytes actually read - public override int Read(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("GZipStream"); - } - var n = BaseStream.Read(buffer, offset, count); - - // Console.WriteLine("GZipStream::Read(buffer, off({0}), c({1}) = {2}", offset, count, n); - // Console.WriteLine( Util.FormatByteArray(buffer, offset, n) ); - - if (!_firstReadDone) - { - _firstReadDone = true; - FileName = BaseStream._GzipFileName; - Comment = BaseStream._GzipComment; - LastModified = BaseStream._GzipMtime; - } - return n; - } - /// /// Calling this method always throws a . /// @@ -328,51 +264,6 @@ public partial class GZipStream : Stream /// irrelevant; this method will always throw! public override void SetLength(long value) => throw new NotSupportedException(); - /// - /// Write data to the stream. - /// - /// - /// - /// - /// If you wish to use the GZipStream to compress data while writing, - /// you can create a GZipStream with CompressionMode.Compress, and a - /// writable output stream. Then call Write() on that GZipStream, - /// providing uncompressed data as input. The data sent to the output stream - /// will be the compressed form of the data written. - /// - /// - /// - /// A GZipStream can be used for Read() or Write(), but not - /// both. Writing implies compression. Reading implies decompression. - /// - /// - /// - /// The buffer holding data to write to the stream. - /// the offset within that data array to find the first byte to write. - /// the number of bytes to write. - public override void Write(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("GZipStream"); - } - if (BaseStream._streamMode == ZlibBaseStream.StreamMode.Undefined) - { - //Console.WriteLine("GZipStream: First write"); - if (BaseStream._wantCompress) - { - // first write in compression, therefore, emit the GZIP header - _headerByteCount = EmitHeader(); - } - else - { - throw new ArchiveOperationException(); - } - } - - BaseStream.Write(buffer, offset, count); - } - #endregion Stream methods public string? Comment @@ -509,13 +400,7 @@ public partial class GZipStream : Stream return header; } - private int EmitHeader() - { - var header = BuildHeader(); - BaseStream._stream.Write(header, 0, header.Length); - return header.Length; // bytes written - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask EmitHeaderAsync(CancellationToken cancellationToken) { var header = BuildHeader(); diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 04ac63cf..4815ec33 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -47,7 +47,7 @@ internal enum ZlibStreamFlavor GZIP = 1952, } -internal class ZlibBaseStream : Stream, IStreamStack +internal partial class ZlibBaseStream : Stream, IStreamStack { Stream IStreamStack.BaseStream() => _stream; @@ -139,58 +139,7 @@ internal class ZlibBaseStream : Stream, IStreamStack private byte[] workingBuffer => _workingBuffer ??= ArrayPool.Shared.Rent(_bufferSize); - public override void Write(byte[] buffer, int offset, int count) - { - // workitem 7159 - // calculate the CRC on the unccompressed data (before writing) - if (crc != null) - { - crc.SlurpBlock(buffer, offset, count); - } - - if (_streamMode == StreamMode.Undefined) - { - _streamMode = StreamMode.Writer; - } - else if (_streamMode != StreamMode.Writer) - { - throw new ZlibException("Cannot Write after Reading."); - } - - if (count == 0) - { - return; - } - - // first reference of z property will initialize the private var _z - z.InputBuffer = buffer; - _z.NextIn = offset; - _z.AvailableBytesIn = count; - var done = false; - do - { - _z.OutputBuffer = workingBuffer; - _z.NextOut = 0; - _z.AvailableBytesOut = _workingBuffer.Length; - var rc = (_wantCompress) ? _z.Deflate(_flushMode) : _z.Inflate(_flushMode); - if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) - { - throw new ZlibException((_wantCompress ? "de" : "in") + "flating: " + _z.Message); - } - - //if (_workingBuffer.Length - _z.AvailableBytesOut > 0) - _stream.Write(_workingBuffer, 0, _workingBuffer.Length - _z.AvailableBytesOut); - - done = _z.AvailableBytesIn == 0 && _z.AvailableBytesOut != 0; - - // If GZIP and de-compress, we're done when 8 bytes remain. - if (_flavor == ZlibStreamFlavor.GZIP && !_wantCompress) - { - done = (_z.AvailableBytesIn == 8 && _z.AvailableBytesOut != 0); - } - } while (!done); - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, @@ -607,27 +556,7 @@ internal class ZlibBaseStream : Stream, IStreamStack _workingBuffer = null; } - public override void Flush() - { - // Only flush the underlying stream when in write mode - // Flushing input streams during read operations is not meaningful - // and can cause issues with forward-only/non-seekable streams - if (_streamMode == StreamMode.Writer) - { - _stream.Flush(); - } - else if (z.AvailableBytesIn > 0) - { - // Rewind the underlying stream by the number of unconsumed bytes in the buffer - // This handles the case where the decompressor over-read past the end of the entry - if (_stream is IStreamStack stack) - { - stack.RewindBytes(z.AvailableBytesIn); - } - z.AvailableBytesIn = 0; - } - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task FlushAsync(CancellationToken cancellationToken) { // Only flush the underlying stream when in write mode @@ -670,31 +599,7 @@ internal class ZlibBaseStream : Stream, IStreamStack private bool nomoreinput; private bool isDisposed; - private string ReadZeroTerminatedString() - { - var list = new List(); - var done = false; - do - { - // workitem 7740 - var n = _stream.Read(_buf1, 0, 1); - if (n != 1) - { - throw new ZlibException("Unexpected EOF reading GZIP header."); - } - if (_buf1[0] == 0) - { - done = true; - } - else - { - list.Add(_buf1[0]); - } - } while (!done); - var buffer = list.ToArray(); - return _encoding.GetString(buffer, 0, buffer.Length); - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask ReadZeroTerminatedStringAsync( CancellationToken cancellationToken ) @@ -844,192 +749,7 @@ internal class ZlibBaseStream : Stream, IStreamStack return totalBytesRead; } - public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) - { - // According to MS documentation, any implementation of the IO.Stream.Read function must: - // (a) throw an exception if offset & count reference an invalid part of the buffer, - // or if count < 0, or if buffer is null - // (b) return 0 only upon EOF, or if count = 0 - // (c) if not EOF, then return at least 1 byte, up to bytes - - if (_streamMode == StreamMode.Undefined) - { - if (!_stream.CanRead) - { - throw new ZlibException("The stream is not readable."); - } - - // for the first read, set up some controls. - _streamMode = StreamMode.Reader; - - // (The first reference to _z goes through the private accessor which - // may initialize it.) - z.AvailableBytesIn = 0; - if (_flavor == ZlibStreamFlavor.GZIP) - { - _gzipHeaderByteCount = _ReadAndValidateGzipHeader(); - - // workitem 8501: handle edge case (decompress empty stream) - if (_gzipHeaderByteCount == 0) - { - return 0; - } - } - } - - if (_streamMode != StreamMode.Reader) - { - throw new ZlibException("Cannot Read after Writing."); - } - - var rc = 0; - - // set up the output of the deflate/inflate codec: - _z.OutputBuffer = buffer; - _z.NextOut = offset; - _z.AvailableBytesOut = count; - - if (count == 0) - { - return 0; - } - if (nomoreinput && _wantCompress) - { - // no more input data available; therefore we flush to - // try to complete the read - rc = _z.Deflate(FlushType.Finish); - - if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) - { - throw new ZlibException( - String.Format( - Constants.DefaultCultureInfo, - "Deflating: rc={0} msg={1}", - rc, - _z.Message - ) - ); - } - - rc = (count - _z.AvailableBytesOut); - - // calculate CRC after reading - if (crc != null) - { - crc.SlurpBlock(buffer, offset, rc); - } - - return rc; - } - ThrowHelper.ThrowIfNull(buffer); - ThrowHelper.ThrowIfNegative(count); - ThrowHelper.ThrowIfLessThan(offset, buffer.GetLowerBound(0)); - if ((offset + count) > buffer.GetLength(0)) - { - throw new ArgumentOutOfRangeException(nameof(count)); - } - - // This is necessary in case _workingBuffer has been resized. (new byte[]) - // (The first reference to _workingBuffer goes through the private accessor which - // may initialize it.) - _z.InputBuffer = workingBuffer; - - do - { - // need data in _workingBuffer in order to deflate/inflate. Here, we check if we have any. - if ((_z.AvailableBytesIn == 0) && (!nomoreinput)) - { - // No data available, so try to Read data from the captive stream. - _z.NextIn = 0; - _z.AvailableBytesIn = _stream.Read(_workingBuffer, 0, _workingBuffer.Length); - if (_z.AvailableBytesIn == 0) - { - nomoreinput = true; - } - } - - // we have data in InputBuffer; now compress or decompress as appropriate - rc = (_wantCompress) ? _z.Deflate(_flushMode) : _z.Inflate(_flushMode); - - if (nomoreinput && (rc == ZlibConstants.Z_BUF_ERROR)) - { - return 0; - } - - if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) - { - throw new ZlibException( - String.Format( - Constants.DefaultCultureInfo, - "{0}flating: rc={1} msg={2}", - (_wantCompress ? "de" : "in"), - rc, - _z.Message - ) - ); - } - - if ( - (nomoreinput || rc == ZlibConstants.Z_STREAM_END) && (_z.AvailableBytesOut == count) - ) - { - break; // nothing more to read - } - } //while (_z.AvailableBytesOut == count && rc == ZlibConstants.Z_OK); - while (_z.AvailableBytesOut > 0 && !nomoreinput && rc == ZlibConstants.Z_OK); - - // workitem 8557 - // is there more room in output? - if (_z.AvailableBytesOut > 0) - { - if (rc == ZlibConstants.Z_OK && _z.AvailableBytesIn == 0) - { - // deferred - } - - // are we completely done reading? - if (nomoreinput) - { - // and in compression? - if (_wantCompress) - { - // no more input data available; therefore we flush to - // try to complete the read - rc = _z.Deflate(FlushType.Finish); - - if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) - { - throw new ZlibException( - String.Format( - Constants.DefaultCultureInfo, - "Deflating: rc={0} msg={1}", - rc, - _z.Message - ) - ); - } - } - } - } - - rc = (count - _z.AvailableBytesOut); - - // calculate CRC after reading - if (crc != null) - { - crc.SlurpBlock(buffer, offset, rc); - } - - if (rc == ZlibConstants.Z_STREAM_END && z.AvailableBytesIn != 0 && !_wantCompress) - { - //rewind the buffer - this.RewindBytes(z.AvailableBytesIn); - z.AvailableBytesIn = 0; - } - - return rc; - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Compressors/Deflate/ZlibStream.Async.cs b/src/SharpCompress/Compressors/Deflate/ZlibStream.Async.cs index 0dee2197..058bd28e 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibStream.Async.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibStream.Async.cs @@ -8,6 +8,10 @@ namespace SharpCompress.Compressors.Deflate; public partial class ZlibStream { + /// + /// Flush the stream. + /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task FlushAsync(CancellationToken cancellationToken) { if (_disposed) @@ -33,6 +37,18 @@ public partial class ZlibStream } #endif + /// + /// Read data from the stream. + /// + /// + /// A ZlibStream compresses or decompresses according to its + /// CompressionMode. It can be used for Read() or Write(), but not both. + /// + /// The buffer into which the read data should be placed. + /// The offset within that data array to put the first byte read. + /// The number of bytes to read. + /// The number of bytes actually read. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, @@ -63,6 +79,17 @@ public partial class ZlibStream } #endif + /// + /// Write data to the stream. + /// + /// + /// A ZlibStream compresses or decompresses according to its + /// CompressionMode. It can be used for Read() or Write(), but not both. + /// + /// The buffer holding data to write to the stream. + /// The offset within that data array to find the first byte to write. + /// The number of bytes to write. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs index c4652d44..580961d7 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs @@ -28,8 +28,6 @@ using System; using System.IO; using System.Text; -using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Compressors.Deflate; @@ -231,53 +229,6 @@ public partial class ZlibStream : Stream } } - /// - /// Flush the stream. - /// - public override void Flush() - { - if (_disposed) - { - throw new ObjectDisposedException("ZlibStream"); - } - _baseStream.Flush(); - } - - /// - /// Read data from the stream. - /// - /// - /// - /// - /// - /// If you wish to use the ZlibStream to compress data while reading, - /// you can create a ZlibStream with CompressionMode.Compress, - /// providing an uncompressed data stream. Then call Read() on that - /// ZlibStream, and the data read will be compressed. If you wish to - /// use the ZlibStream to decompress data while reading, you can create - /// a ZlibStream with CompressionMode.Decompress, providing a - /// readable compressed data stream. Then call Read() on that - /// ZlibStream, and the data will be decompressed as it is read. - /// - /// - /// - /// A ZlibStream can be used for Read() or Write(), but - /// not both. - /// - /// - /// - /// The buffer into which the read data should be placed. - /// the offset within that data array to put the first byte read. - /// the number of bytes to read. - public override int Read(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("ZlibStream"); - } - return _baseStream.Read(buffer, offset, count); - } - public override int ReadByte() { if (_disposed) @@ -297,41 +248,6 @@ public partial class ZlibStream : Stream /// public override void SetLength(long value) => throw new NotSupportedException(); - /// - /// Write data to the stream. - /// - /// - /// - /// - /// - /// If you wish to use the ZlibStream to compress data while writing, - /// you can create a ZlibStream with CompressionMode.Compress, - /// and a writable output stream. Then call Write() on that - /// ZlibStream, providing uncompressed data as input. The data sent to - /// the output stream will be the compressed form of the data written. If you - /// wish to use the ZlibStream to decompress data while writing, you - /// can create a ZlibStream with CompressionMode.Decompress, and a - /// writable output stream. Then call Write() on that stream, - /// providing previously compressed data. The data sent to the output stream - /// will be the decompressed form of the data written. - /// - /// - /// - /// A ZlibStream can be used for Read() or Write(), but not both. - /// - /// - /// The buffer holding data to write to the stream. - /// the offset within that data array to find the first byte to write. - /// the number of bytes to write. - public override void Write(byte[] buffer, int offset, int count) - { - if (_disposed) - { - throw new ObjectDisposedException("ZlibStream"); - } - _baseStream.Write(buffer, offset, count); - } - public override void WriteByte(byte value) { if (_disposed) diff --git a/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.Async.cs b/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.Async.cs index 64652872..dbb8afa6 100644 --- a/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.Async.cs +++ b/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.Async.cs @@ -10,6 +10,7 @@ namespace SharpCompress.Compressors.Deflate64; public sealed partial class Deflate64Stream { + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task ReadAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.cs b/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.cs index 9e9ff7db..8d06436f 100644 --- a/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.cs +++ b/src/SharpCompress/Compressors/Deflate64/Deflate64Stream.cs @@ -5,10 +5,7 @@ using System; using System.IO; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common; -using SharpCompress.Common.Zip; namespace SharpCompress.Compressors.Deflate64; @@ -69,50 +66,6 @@ public sealed partial class Deflate64Stream : Stream public override void SetLength(long value) => throw new NotSupportedException("Deflate64: not supported"); - public override int Read(byte[] buffer, int offset, int count) - { - ValidateParameters(buffer, offset, count); - EnsureNotDisposed(); - - int bytesRead; - var currentOffset = offset; - var remainingCount = count; - - while (true) - { - bytesRead = _inflater.Inflate(buffer, currentOffset, remainingCount); - currentOffset += bytesRead; - remainingCount -= bytesRead; - - if (remainingCount == 0) - { - break; - } - - if (_inflater.Finished()) - { - // if we finished decompressing, we can't have anything left in the outputwindow. - break; - } - - var bytes = _stream.Read(_buffer, 0, _buffer.Length); - if (bytes <= 0) - { - break; - } - else if (bytes > _buffer.Length) - { - // The stream is either malicious or poorly implemented and returned a number of - // bytes larger than the buffer supplied to it. - throw new InvalidFormatException("Deflate64: invalid data"); - } - - _inflater.SetInput(_buffer, 0, bytes); - } - - return count - remainingCount; - } - private void ValidateParameters(byte[] array, int offset, int count) { ThrowHelper.ThrowIfNull(array); diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs index 78300446..2b38fac7 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzOutWindow.cs @@ -24,6 +24,35 @@ internal partial class OutWindow : IDisposable public long Total => _total; +#if !LEGACY_DOTNET + // Fast-path accessors used by the local-variable LZMA decode loop (see + // LzmaDecoder.Fast.cs). CodeFast snapshots pos/total/buffer into locals for the whole + // decode call instead of going through PutByte/GetByte/CopyBlock (and re-reading the + // fields of this object) for every single output byte, mirroring how the reference + // 7-Zip C decoder caches dicPos/dic as locals for the duration of one decode call. + internal byte[] FastBuffer => _buffer; + internal int FastPos + { + get => _pos; + set => _pos = value; + } + internal long FastTotal + { + get => _total; + set => _total = value; + } + internal int FastWindowSize => _windowSize; + internal long FastLimit => _limit; + + internal void FastFlush() => Flush(); + + internal void SetPendingFast(int distance, int len) + { + _pendingDist = distance; + _pendingLen = len; + } +#endif + public void Create(int windowSize) { if (windowSize <= 0) diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs b/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs new file mode 100644 index 00000000..cb63751b --- /dev/null +++ b/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs @@ -0,0 +1,437 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +#if !LEGACY_DOTNET +using Microsoft.Win32.SafeHandles; +#endif + +namespace SharpCompress.Compressors.LZMA; + +/// +/// A contiguous run of one or more independent LZMA2 chunks that can be decoded without any +/// state (dictionary / range-coder) carried over from a preceding chunk. +/// +internal readonly record struct Lzma2Block( + long PackOffset, + long PackLen, + long UnpackOffset, + long UnpackLen +); + +/// +/// Splits a single LZMA2-coded stream into independently-decodable blocks and decodes them +/// concurrently. The block-splitting strategy mirrors 7-Zip's own multi-threaded LZMA2 decoder +/// (see the reference 7-Zip C source, Lzma2DecMt.c / MtDec.c): independent restart points +/// (LZMA2 control byte 0x01, or 0xE0-0xFF) are candidate block boundaries, but consecutive +/// restart segments are merged together until the accumulated block reaches at least +/// decoded bytes -- 7-Zip's own comment for this is "we decode small +/// blocks in one thread" -- to avoid parallelization overhead for tiny fragments. Merging also +/// stops once a block would exceed , matching 7-Zip's outBlockMax +/// default (Lzma2DecMtProps_Init: 1<<28 / 256MB). +/// +internal static class Lzma2ParallelDecoder +{ + // 7-Zip Lzma2DecMt.c (block-parse loop): "we decode small blocks in one thread" -- + // if (t->dec.decoder.dicPos >= (1 << 14)) break; + internal const int MinBlockSize = 1 << 14; + + // 7-Zip Lzma2DecMt.c: Lzma2DecMtProps_Init -> p->outBlockMax = 1 << 28 + internal const long MaxBlockSize = 1 << 28; + + // 7-Zip MtDec.h: #define MTDEC_THREADS_MAX 32 + internal const int MaxThreads = 32; + + // Bound each worker's transient buffers independently of the compressed block size. + internal const int BlockInputBufferSize = 1 << 16; + internal const int BlockOutputBufferSize = 1 << 20; + + /// + /// Header-only walk of the LZMA2 chunk stream (no LZMA decoding) that locates every + /// independent restart point and merges adjacent segments into blocks using the same size + /// thresholds 7-Zip's own MT decoder uses. Returns null if the stream doesn't parse as + /// well-formed LZMA2 or the parsed total doesn't match ; callers + /// should fall back to ordinary sequential decoding in that case. + /// + internal static List? TryScanBlocks( + Stream packStream, + long packStart, + long packSize, + long unpackSize + ) + { + try + { + packStream.Position = packStart; + + var blocks = new List(); + long packPos = 0; + long unpackPos = 0; + + var blockPackStart = 0L; + var blockUnpackStart = 0L; + var blockUnpackSize = 0L; + var blockHasContent = false; + + Span sizeBuf = stackalloc byte[2]; + Span header = stackalloc byte[4]; + + while (packPos < packSize) + { + var chunkPackStart = packPos; + var control = packStream.ReadByte(); + if (control < 0) + { + return null; // truncated stream + } + packPos++; + + if (control == 0) + { + break; // end-of-stream marker + } + + long chunkUnpackSize; + long chunkPackPayload; + bool isRestart; + + if (control < 0x80) + { + // 0x01 = uncompressed chunk + dictionary reset, 0x02 = uncompressed, no reset + if (control > 2) + { + return null; // not a recognized LZMA2 control byte + } + if (!ReadFully(packStream, sizeBuf)) + { + return null; + } + packPos += 2; + chunkUnpackSize = ((sizeBuf[0] << 8) | sizeBuf[1]) + 1; + chunkPackPayload = chunkUnpackSize; + isRestart = control == 1; + } + else + { + // 0x80-0xFF = compressed chunk; bit layout: 1uuuuu, u = high bits of unpack size + if (!ReadFully(packStream, header)) + { + return null; + } + packPos += 4; + chunkUnpackSize = + (long)(((control & 0x1F) << 16) | (header[0] << 8) | header[1]) + 1; + chunkPackPayload = ((header[2] << 8) | header[3]) + 1; + // 0xA0-0xBF state reset, 0xC0-0xDF state reset + new props, 0xE0-0xFF also + // resets the dictionary -- only a dictionary reset is an independent restart. + isRestart = control >= 0xE0; + + // 0xC0-0xFF (new-props flag, bits 6-5 = 1x) carries one extra properties byte + // right after the 4 size-field bytes, before the compressed payload begins. + if (control >= 0xC0) + { + if (packStream.ReadByte() < 0) + { + return null; + } + packPos++; + } + } + + if ( + isRestart + && blockHasContent + && ( + blockUnpackSize >= MinBlockSize + || blockUnpackSize + chunkUnpackSize > MaxBlockSize + ) + ) + { + blocks.Add( + new Lzma2Block( + blockPackStart, + chunkPackStart - blockPackStart, + blockUnpackStart, + blockUnpackSize + ) + ); + blockPackStart = chunkPackStart; + blockUnpackStart = unpackPos; + blockUnpackSize = 0; + blockHasContent = false; + } + + packPos += chunkPackPayload; + if (packPos > packSize) + { + return null; // chunk claims more pack bytes than the folder actually has + } + packStream.Seek(chunkPackPayload, SeekOrigin.Current); + unpackPos += chunkUnpackSize; + blockUnpackSize += chunkUnpackSize; + blockHasContent = true; + } + + if (unpackPos != unpackSize || packPos != packSize) + { + return null; // scan didn't land on the expected totals -- don't trust it + } + + if (blockHasContent) + { + blocks.Add( + new Lzma2Block( + blockPackStart, + packPos - blockPackStart, + blockUnpackStart, + blockUnpackSize + ) + ); + } + + return blocks; + } + catch (Exception) + { + return null; + } + } + + private static bool ReadFully(Stream stream, Span buffer) + { + var total = 0; + while (total < buffer.Length) + { + var read = stream.Read(buffer.Slice(total)); + if (read <= 0) + { + return false; + } + total += read; + } + return true; + } + +#if !LEGACY_DOTNET + /// + /// Decodes every block concurrently, reading packed bytes positionally from + /// and writing decoded bytes positionally to + /// . Both handles must support positional (random) access + /// since blocks complete out of order across threads. + /// + internal static void DecodeBlocksParallel( + SafeFileHandle inputHandle, + byte[] lzma2Props, + long packStart, + IReadOnlyList blocks, + SafeFileHandle outputHandle, + int maxDegreeOfParallelism + ) => + DecodeBlocksParallel( + inputHandle, + lzma2Props, + packStart, + blocks, + outputHandle, + maxDegreeOfParallelism, + ArrayPool.Shared + ); + + internal static void DecodeBlocksParallel( + SafeFileHandle inputHandle, + byte[] lzma2Props, + long packStart, + IReadOnlyList blocks, + SafeFileHandle outputHandle, + int maxDegreeOfParallelism, + ArrayPool bufferPool + ) + { + ThrowHelper.ThrowIfNull(bufferPool); + + Parallel.ForEach( + blocks, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, + block => + { + using var packStream = new RandomAccessBlockStream( + inputHandle, + checked(packStart + block.PackOffset), + block.PackLen, + bufferPool + ); + using var lzma = LzmaStream.Create( + lzma2Props, + packStream, + block.PackLen, + block.UnpackLen, + leaveOpen: true + ); + + var bufferSize = checked( + (int)Math.Min(Math.Max(block.UnpackLen, 1), BlockOutputBufferSize) + ); + var buffer = bufferPool.Rent(bufferSize); + try + { + long total = 0; + while (total < block.UnpackLen) + { + var toRead = (int)Math.Min(bufferSize, block.UnpackLen - total); + var read = lzma.Read(buffer, 0, toRead); + if (read <= 0) + { + throw new EndOfStreamException( + $"LZMA2 block at unpack offset {block.UnpackOffset:N0} ended early after {total:N0}/{block.UnpackLen:N0} bytes." + ); + } + RandomAccess.Write( + outputHandle, + buffer.AsSpan(0, read), + checked(block.UnpackOffset + total) + ); + total += read; + } + } + finally + { + bufferPool.Return(buffer, clearArray: true); + } + } + ); + } + + private sealed class RandomAccessBlockStream : Stream + { + private readonly SafeFileHandle _inputHandle; + private readonly long _start; + private readonly long _length; + private readonly ArrayPool _bufferPool; + private byte[]? _buffer; + private long _bufferStart = -1; + private int _bufferLength; + private long _position; + + internal RandomAccessBlockStream( + SafeFileHandle inputHandle, + long start, + long length, + ArrayPool bufferPool + ) + { + ThrowHelper.ThrowIfNegative(length); + + _inputHandle = inputHandle; + _start = start; + _length = length; + _bufferPool = bufferPool; + _buffer = bufferPool.Rent(BlockInputBufferSize); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => + Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var total = 0; + while (!buffer.IsEmpty && _position < _length) + { + if (!FillBuffer()) + { + break; + } + var bufferOffset = checked((int)(_position - _bufferStart)); + var count = Math.Min(_bufferLength - bufferOffset, buffer.Length); + _buffer!.AsSpan(bufferOffset, count).CopyTo(buffer); + _position += count; + total += count; + buffer = buffer.Slice(count); + } + return total; + } + + public override int ReadByte() + { + if (_position >= _length) + { + return -1; + } + + if (!FillBuffer()) + { + return -1; + } + var value = _buffer![checked((int)(_position - _bufferStart))]; + _position++; + return value; + } + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && _buffer is not null) + { + _bufferPool.Return(_buffer, clearArray: true); + _buffer = null; + } + + base.Dispose(disposing); + } + + private bool FillBuffer() + { + if ( + _bufferStart >= 0 + && _position >= _bufferStart + && _position - _bufferStart < _bufferLength + ) + { + return true; + } + + var buffer = + _buffer ?? throw new ObjectDisposedException(nameof(RandomAccessBlockStream)); + var requested = (int)Math.Min(buffer.Length, _length - _position); + var total = 0; + while (total < requested) + { + var read = RandomAccess.Read( + _inputHandle, + buffer.AsSpan(total, requested - total), + checked(_start + _position + total) + ); + if (read <= 0) + { + break; + } + total += read; + } + + _bufferStart = _position; + _bufferLength = total; + return total > 0; + } + } +#endif +} diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs new file mode 100644 index 00000000..81ad80d8 --- /dev/null +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs @@ -0,0 +1,794 @@ +#nullable disable + +#if !LEGACY_DOTNET +using System; +using System.Runtime.CompilerServices; +using SharpCompress.Compressors.LZMA.LZ; + +namespace SharpCompress.Compressors.LZMA; + +// Fast LZMA decode path used on .NET targets that support it (everything except +// net48/netstandard2.0/netstandard2.1, see LEGACY_DOTNET). This mirrors the design of the +// reference LZMA SDK / 7-Zip C decoder (LzmaDec.c): probabilities are stored in flat ushort +// arrays (instead of arrays of BitDecoder structs reached through nested decoder objects), +// range/code/dictionary-position/input-buffer-position are kept in local variables (pinned +// with `fixed` and accessed through raw pointers, exactly like the C reference's `probs`/ +// `dic`/`buf` locals) for the duration of the decode loop, instead of being re-read from +// object fields - and re-bounds-checked - on every bit. Compressed input is consumed from a +// buffered reader (RangeCoder.Decoder's fast buffer) instead of one virtual Stream.ReadByte() +// call per byte. +// +// These tables are separate from (and not kept in sync with) the BitDecoder-based tables used +// by the async decode path (LzmaDecoder.Async.cs), which remains unchanged on every target. +// Both sets are kept up to date by SetDecoderProperties/Init, so either decode path can be used +// on a given Decoder instance, as long as sync and async decoding are not interleaved on the +// same instance. +public partial class Decoder +{ + private const int KNumMoveBitsFast = 5; + + // Bias used by the branchless probability-update formula in DecodeBitFast (see there for + // the derivation): (1 << KNumMoveBitsFast) - 1. + private const int KBitModelOffsetFast = (1 << KNumMoveBitsFast) - 1; + + // Choice/Choice2 flags followed by Low (16 posStates * 8 symbols), Mid (16 posStates * 8 + // symbols) and High (256 symbols) trees, laid out contiguously rather than reusing 7-Zip's + // overlapping ASM-oriented addressing, to keep the arithmetic straightforward and safe. + private const int LenChoiceIndex = 0; + private const int LenChoice2Index = 1; + private const int LenLowBase = 2; + private const int LenLowStride = 1 << Base.K_NUM_LOW_LEN_BITS; // 8 + private const int LenLowSize = (int)Base.K_NUM_POS_STATES_MAX * LenLowStride; // 128 + private const int LenMidBase = LenLowBase + LenLowSize; + private const int LenMidStride = 1 << Base.K_NUM_MID_LEN_BITS; // 8 + private const int LenMidSize = (int)Base.K_NUM_POS_STATES_MAX * LenMidStride; // 128 + private const int LenHighBase = LenMidBase + LenMidSize; + private const int LenHighSize = 1 << Base.K_NUM_HIGH_LEN_BITS; // 256 + private const int LenProbsSize = LenHighBase + LenHighSize; + + private ushort[] _fIsMatch; + private ushort[] _fIsRep; + private ushort[] _fIsRepG0; + private ushort[] _fIsRepG1; + private ushort[] _fIsRepG2; + private ushort[] _fIsRep0Long; + private ushort[] _fPosSlot; + private ushort[] _fPosDecoders; + private ushort[] _fPosAlign; + private ushort[] _fLenProbs; + private ushort[] _fRepLenProbs; + private ushort[] _fLiteral; + + private int _fLiteralNumPrevBits = -1; + private int _fLiteralNumPosBits = -1; + private uint _fLiteralPosMask; + + private void CreateFastModel(int lp, int lc) + { + _fIsMatch ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; + _fIsRep ??= new ushort[Base.K_NUM_STATES]; + _fIsRepG0 ??= new ushort[Base.K_NUM_STATES]; + _fIsRepG1 ??= new ushort[Base.K_NUM_STATES]; + _fIsRepG2 ??= new ushort[Base.K_NUM_STATES]; + _fIsRep0Long ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; + _fPosSlot ??= new ushort[Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS]; + _fPosDecoders ??= new ushort[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX]; + _fPosAlign ??= new ushort[1 << Base.K_NUM_ALIGN_BITS]; + _fLenProbs ??= new ushort[LenProbsSize]; + _fRepLenProbs ??= new ushort[LenProbsSize]; + + if (_fLiteralNumPrevBits != lc || _fLiteralNumPosBits != lp) + { + _fLiteralNumPrevBits = lc; + _fLiteralNumPosBits = lp; + _fLiteralPosMask = ((uint)1 << lp) - 1; + var numStates = (uint)1 << (lc + lp); + _fLiteral = new ushort[checked((int)(numStates * 0x300))]; + } + } + + private void InitFastModel() + { + const ushort probInit = (ushort)(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL >> 1); + Array.Fill(_fIsMatch, probInit); + Array.Fill(_fIsRep, probInit); + Array.Fill(_fIsRepG0, probInit); + Array.Fill(_fIsRepG1, probInit); + Array.Fill(_fIsRepG2, probInit); + Array.Fill(_fIsRep0Long, probInit); + Array.Fill(_fPosSlot, probInit); + Array.Fill(_fPosDecoders, probInit); + Array.Fill(_fPosAlign, probInit); + Array.Fill(_fLenProbs, probInit); + Array.Fill(_fRepLenProbs, probInit); + Array.Fill(_fLiteral, probInit); + } + + // Mutable range-coder state threaded through the decode helpers below as a single `ref` + // parameter (instead of one `ref` per field), pinned to the RangeCoder.Decoder's fast + // input buffer for the duration of one CodeFast call. Consumed byte count is batched into + // RangeCoder.Decoder._total only when the local buffer is refilled/at call exit, instead + // of touching that field on every single byte. + private unsafe struct FastRangeState + { + public uint Range; + public uint Code; + public byte* InBuf; + public int InPos; + public int InLen; + public long Consumed; + public RangeCoder.Decoder RangeDecoder; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte ReadByte() + { + if (InPos >= InLen) + { + RangeDecoder.FastBufferPos = InPos; + RangeDecoder.AddTotal(Consumed); + Consumed = 0; + RangeDecoder.RefillFast(); + InPos = RangeDecoder.FastBufferPos; + InLen = RangeDecoder.FastBufferLen; + } + Consumed++; + return InBuf[InPos++]; + } + } + + // Mutable dictionary/output-window state, pinned to the OutWindow's circular buffer for + // the duration of one CodeFast call. Mirrors dicPos/dic being plain locals in the + // reference 7-Zip C decoder instead of being re-read from the OutWindow object. + private unsafe struct FastOutState + { + public byte* Dic; + public int Pos; + public long Total; + public int WindowSize; + public long Limit; + public OutWindow OutWindow; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly byte GetByte(int distance) + { + var p = Pos - distance - 1; + if (p < 0) + { + p += WindowSize; + } + return Dic[p]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PutByte(byte value) + { + Dic[Pos++] = value; + Total++; + if (Pos >= WindowSize) + { + // Rare (once per dictionary-size worth of output): sync locals back, let + // OutWindow flush the pending bytes out to the underlying stream and wrap + // its position, then reload the (now wrapped) position. + OutWindow.FastPos = Pos; + OutWindow.FastTotal = Total; + OutWindow.FastFlush(); + Pos = OutWindow.FastPos; + } + } + + public void CopyBlock(int distance, int len) + { + var rem = len; + var p = (distance < Pos ? Pos : Pos + WindowSize) - distance - 1; + var targetSize = + (Pos < WindowSize && Total < Limit) ? (int)Math.Min(rem, Limit - Total) : 0; + var sizeUntilWindowEnd = Math.Min(WindowSize - Pos, WindowSize - p); + var sizeUntilOverlap = Math.Abs(p - Pos); + var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize); + if (fastSize >= 2) + { + Buffer.MemoryCopy(Dic + p, Dic + Pos, fastSize, fastSize); + Pos += fastSize; + p += fastSize; + Total += fastSize; + if (Pos >= WindowSize) + { + OutWindow.FastPos = Pos; + OutWindow.FastTotal = Total; + OutWindow.FastFlush(); + Pos = OutWindow.FastPos; + } + rem -= fastSize; + } + while (rem > 0 && Pos < WindowSize && Total < Limit) + { + if (p >= WindowSize) + { + p = 0; + } + PutByte(Dic[p++]); + rem--; + } + OutWindow.SetPendingFast(distance, rem); + } + } + + internal unsafe bool CodeFast( + int dictionarySize, + OutWindow outWindow, + RangeCoder.Decoder rangeDecoder + ) + { + var dictionarySizeCheck = Math.Max(dictionarySize, 1); + + outWindow.CopyPending(); + + var dic = outWindow.FastBuffer; + var inBuf = rangeDecoder.FastBufferArray; + bool result; + + fixed (byte* pDic = dic) + fixed (byte* pIn = inBuf) + fixed ( + ushort* pIsMatch = _fIsMatch, + pIsRep = _fIsRep, + pIsRepG0 = _fIsRepG0, + pIsRepG1 = _fIsRepG1, + pIsRepG2 = _fIsRepG2, + pIsRep0Long = _fIsRep0Long, + pPosSlot = _fPosSlot, + pPosDecoders = _fPosDecoders, + pPosAlign = _fPosAlign, + pLenProbs = _fLenProbs, + pRepLenProbs = _fRepLenProbs, + pLiteral = _fLiteral + ) + { + var rs = new FastRangeState + { + Range = rangeDecoder._range, + Code = rangeDecoder._code, + InBuf = pIn, + InPos = rangeDecoder.FastBufferPos, + InLen = rangeDecoder.FastBufferLen, + Consumed = 0, + RangeDecoder = rangeDecoder, + }; + var os = new FastOutState + { + Dic = pDic, + Pos = outWindow.FastPos, + Total = outWindow.FastTotal, + WindowSize = outWindow.FastWindowSize, + Limit = outWindow.FastLimit, + OutWindow = outWindow, + }; + + result = false; + while (os.Pos < os.WindowSize && os.Total < os.Limit) + { + var posState = (uint)os.Total & _posStateMask; + var stateIndex = (int)_state._index; + // (stateIndex << K_NUM_POS_STATES_BITS_MAX) + posState is used both as the + // IsMatch index and (numerically identical) the IsRep0Long/"short rep" index. + var matchIndex = (int)((stateIndex << Base.K_NUM_POS_STATES_BITS_MAX) + posState); + + // prevByte/matchByte only depend on os.Pos/_rep0, both already fixed from the + // previous iteration - independent of the upcoming (unavoidably branchy, since + // it changes the whole loop-body shape) IsMatch dispatch below. Issuing these + // dictionary reads before that branch resolves lets their load latency overlap + // with the IsMatch bit-decode arithmetic instead of stalling right after the + // branch, where they'd otherwise sit on the critical path. (Tried hoisting the + // literal-tree base-index/root-probability computation the same way too, but it + // regressed ~1s - that computation is heavy enough that doing it unconditionally + // on every iteration, including non-literal ones, outweighs the latency-hiding + // benefit; kept lazy, computed only once IsMatch resolves to 0.) + var isCharState = _state.IsCharState(); + var prevByte = os.GetByte(0); + var matchByte = isCharState ? (byte)0 : os.GetByte((int)_rep0); + + // Split compare/update (mirrors ASM's IF_BIT_x_NOUP + deferred UPDATE_0/ + // UPDATE_1): the probability write only happens once we know which arm we're + // in, interleaved with that arm's own independent setup work below, instead of + // being on the decode's own critical path. + var probIsMatch = pIsMatch[matchIndex]; + var isMatchSymbol = DecodeBitFastNoUpdate(ref rs, probIsMatch, out var isMatchMask); + + if (isMatchSymbol == 0) + { + UpdateProbFast(pIsMatch + matchIndex, probIsMatch, isMatchMask); + var literalP = + pLiteral + (int)GetFastLiteralBaseIndex((uint)os.Total, prevByte); + var firstIndex = isCharState + ? 1u + : (uint)((((matchByte >> 7) & 1) + 1) << 8) + 1; + var firstProb = (uint)literalP[firstIndex]; + var b = isCharState + ? LiteralDecodeNormalFast(ref rs, literalP, firstProb) + : LiteralDecodeWithMatchByteFast( + ref rs, + literalP, + matchByte, + firstIndex, + firstProb + ); + os.PutByte(b); + _state.UpdateChar(); + continue; + } + + UpdateProbFast(pIsMatch + matchIndex, probIsMatch, isMatchMask); + + uint len; + if (DecodeBitFast(ref rs, pIsRep, stateIndex) == 1) + { + if (DecodeBitFast(ref rs, pIsRepG0, stateIndex) == 0) + { + if (DecodeBitFast(ref rs, pIsRep0Long, matchIndex) == 0) + { + _state.UpdateShortRep(); + os.PutByte(os.GetByte((int)_rep0)); + continue; + } + } + else + { + uint distance; + if (DecodeBitFast(ref rs, pIsRepG1, stateIndex) == 0) + { + distance = _rep1; + } + else + { + if (DecodeBitFast(ref rs, pIsRepG2, stateIndex) == 0) + { + distance = _rep2; + } + else + { + distance = _rep3; + _rep3 = _rep2; + } + _rep2 = _rep1; + } + _rep1 = _rep0; + _rep0 = distance; + } + len = LenDecodeFast(ref rs, pRepLenProbs, posState) + Base.K_MATCH_MIN_LEN; + _state.UpdateRep(); + } + else + { + _rep3 = _rep2; + _rep2 = _rep1; + _rep1 = _rep0; + len = Base.K_MATCH_MIN_LEN + LenDecodeFast(ref rs, pLenProbs, posState); + _state.UpdateMatch(); + var posSlot = BitTreeDecodeFast( + ref rs, + pPosSlot, + (int)(Base.GetLenToPosState(len) << Base.K_NUM_POS_SLOT_BITS), + Base.K_NUM_POS_SLOT_BITS + ); + if (posSlot >= Base.K_START_POS_MODEL_INDEX) + { + var numDirectBits = (int)((posSlot >> 1) - 1); + _rep0 = (2 | (posSlot & 1)) << numDirectBits; + if (posSlot < Base.K_END_POS_MODEL_INDEX) + { + _rep0 += BitTreeReverseDecodeFast( + ref rs, + pPosDecoders, + (int)(_rep0 - posSlot - 1), + numDirectBits + ); + } + else + { + _rep0 += + DecodeDirectBitsFast(ref rs, numDirectBits - Base.K_NUM_ALIGN_BITS) + << Base.K_NUM_ALIGN_BITS; + _rep0 += BitTreeReverseDecodeFast( + ref rs, + pPosAlign, + 0, + Base.K_NUM_ALIGN_BITS + ); + } + } + else + { + _rep0 = posSlot; + } + } + + if (_rep0 >= os.Total || _rep0 >= dictionarySizeCheck) + { + if (_rep0 == 0xFFFFFFFF) + { + result = true; + break; + } + rangeDecoder._range = rs.Range; + rangeDecoder._code = rs.Code; + rangeDecoder.FastBufferPos = rs.InPos; + rangeDecoder.AddTotal(rs.Consumed); + outWindow.FastPos = os.Pos; + outWindow.FastTotal = os.Total; + throw new DataErrorException(); + } + os.CopyBlock((int)_rep0, (int)len); + } + + rangeDecoder._range = rs.Range; + rangeDecoder._code = rs.Code; + rangeDecoder.FastBufferPos = rs.InPos; + rangeDecoder.AddTotal(rs.Consumed); + outWindow.FastPos = os.Pos; + outWindow.FastTotal = os.Total; + } + + return result; + } + + // Core of DecodeBitFast, factored out so the bit-tree/literal decoders below can pass in a + // probability that was already loaded (prefetched) by the *previous* tree level instead of + // loading it here, and can read back `mask` to select their own prefetched next-level + // probability without an extra branch. `probSlot` must point at the array slot the caller + // read `prob` from, for the write-back. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint DecodeBitFastCore( + ref FastRangeState rs, + ushort* probSlot, + uint prob, + out uint mask + ) + { + var range = rs.Range; + var bound = (range >> RangeCoder.BitDecoder.K_NUM_BIT_MODEL_TOTAL_BITS) * prob; + + // Branchless bit decode + probability update, mirroring 7-Zip's ASM decoder + // (Asm/x86/LzmaDecOpt.asm) rather than the reference C decoder's data-dependent + // `if`: both possible next-states are computed unconditionally and combined via an + // all-ones/all-zeros mask (the software equivalent of the ASM's cmovae/cmovb), + // instead of branching on a comparison whose outcome is close to 50/50 for + // well-compressed data and is therefore poorly predicted by the CPU. + var symbol = rs.Code < bound ? 0u : 1u; + mask = (uint)-(int)symbol; // 0 for symbol 0, 0xFFFFFFFF for symbol 1 + + rs.Range = (bound & ~mask) | ((range - bound) & mask); + rs.Code -= bound & mask; + + // UPDATE_0 (symbol 0) is `prob + ((K_BIT_MODEL_TOTAL - prob) >> 5)`; UPDATE_1 + // (symbol 1) is `prob - (prob >> 5)`. KBitModelOffsetFast (31) is the standard LZMA + // bias that makes a single *arithmetic* (signed) shift reproduce UPDATE_1's + // logical-shift result from a target of 0, so both formulas collapse into one + // branchless expression selected by the same mask used above. + int target = (int)( + (RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL & ~mask) | (uint)(KBitModelOffsetFast & mask) + ); + *probSlot = (ushort)((int)prob + ((target - (int)prob) >> KNumMoveBitsFast)); + + if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE) + { + rs.Range <<= 8; + rs.Code = (rs.Code << 8) | rs.ReadByte(); + } + return symbol; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint DecodeBitFast(ref FastRangeState rs, ushort* probs, int index) => + DecodeBitFastCore(ref rs, probs + index, probs[index], out _); + + // Split form of DecodeBitFastCore used only for the outer dispatch bit (IsMatch): mirrors + // the ASM's IF_BIT_x_NOUP / UPDATE_0 / UPDATE_1 split, where the probability-model update is + // deferred past the (unavoidable, since literal vs. match are wholly different code) branch + // so it can be interleaved with independent work in each arm instead of sitting on the + // decode's own critical path. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint DecodeBitFastNoUpdate(ref FastRangeState rs, uint prob, out uint mask) + { + var range = rs.Range; + var bound = (range >> RangeCoder.BitDecoder.K_NUM_BIT_MODEL_TOTAL_BITS) * prob; + var symbol = rs.Code < bound ? 0u : 1u; + mask = (uint)-(int)symbol; + rs.Range = (bound & ~mask) | ((range - bound) & mask); + rs.Code -= bound & mask; + if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE) + { + rs.Range <<= 8; + rs.Code = (rs.Code << 8) | rs.ReadByte(); + } + return symbol; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void UpdateProbFast(ushort* probSlot, uint prob, uint mask) + { + int target = (int)( + (RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL & ~mask) | (uint)(KBitModelOffsetFast & mask) + ); + *probSlot = (ushort)((int)prob + ((target - (int)prob) >> KNumMoveBitsFast)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint BitTreeDecodeFast( + ref FastRangeState rs, + ushort* probs, + int baseIndex, + int numBits + ) + { + var p = probs + baseIndex; + uint m = 1; + // The very first probability must still be loaded fresh; from then on `prob` always + // arrives pre-loaded from the previous level's prefetch below. + var prob = (uint)p[1]; + for (var i = 0; i < numBits; i++) + { + var child0 = m << 1; + // Prefetch BOTH possible next-level probabilities now: which child we'll actually + // need depends on the bit we're about to decode, but the child *indices* only + // depend on `m`, which is already known - so these loads can proceed in parallel + // with (instead of only after) the branchless decode below, hiding their latency. + var hasNext = i + 1 < numBits; + uint probChild0 = 0, + probChild1 = 0; + if (hasNext) + { + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + } + + var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask); + + m = child0 + bit; + if (hasNext) + { + prob = (probChild0 & ~mask) | (probChild1 & mask); + } + } + return m - ((uint)1 << numBits); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint BitTreeReverseDecodeFast( + ref FastRangeState rs, + ushort* probs, + int baseIndex, + int numBits + ) + { + var p = probs + baseIndex; + uint m = 1; + var prob = (uint)p[1]; + uint symbol = 0; + for (var i = 0; i < numBits; i++) + { + var child0 = m << 1; + var hasNext = i + 1 < numBits; + uint probChild0 = 0, + probChild1 = 0; + if (hasNext) + { + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + } + + var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask); + symbol |= bit << i; + + m = child0 + bit; + if (hasNext) + { + prob = (probChild0 & ~mask) | (probChild1 & mask); + } + } + return symbol; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint DecodeDirectBitsFast(ref FastRangeState rs, int numTotalBits) + { + uint result = 0; + for (var i = numTotalBits; i > 0; i--) + { + rs.Range >>= 1; + var t = (rs.Code - rs.Range) >> 31; + rs.Code -= rs.Range & (t - 1); + result = (result << 1) | (1 - t); + if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE) + { + rs.Code = (rs.Code << 8) | rs.ReadByte(); + rs.Range <<= 8; + } + } + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint LenDecodeFast(ref FastRangeState rs, ushort* probs, uint posState) + { + if (DecodeBitFast(ref rs, probs, LenChoiceIndex) == 0) + { + return BitTreeDecodeFast( + ref rs, + probs, + LenLowBase + (int)posState * LenLowStride, + Base.K_NUM_LOW_LEN_BITS + ); + } + var symbol = Base.K_NUM_LOW_LEN_SYMBOLS; + if (DecodeBitFast(ref rs, probs, LenChoice2Index) == 0) + { + symbol += BitTreeDecodeFast( + ref rs, + probs, + LenMidBase + (int)posState * LenMidStride, + Base.K_NUM_MID_LEN_BITS + ); + } + else + { + symbol += Base.K_NUM_MID_LEN_SYMBOLS; + symbol += BitTreeDecodeFast(ref rs, probs, LenHighBase, Base.K_NUM_HIGH_LEN_BITS); + } + return symbol; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private uint GetFastLiteralBaseIndex(uint pos, byte prevByte) => + ( + ((pos & _fLiteralPosMask) << _fLiteralNumPrevBits) + + (uint)(prevByte >> (8 - _fLiteralNumPrevBits)) + ) * 0x300; + + // baseP/firstProb (root of the literal tree, index 1) are precomputed by the caller - + // pos/prevByte are already known before the IsMatch decode resolves, so hoisting this load + // out lets it overlap with the IsMatch bit-decode instead of stalling right after it. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe byte LiteralDecodeNormalFast( + ref FastRangeState rs, + ushort* baseP, + uint firstProb + ) + { + var p = baseP; + // Same one-level-ahead child-probability prefetch as BitTreeDecodeFast (see there), + // applied to the hottest call site in the decoder: one 8-bit literal-tree decode per + // literal byte. Manually unrolled (mirroring the ASM's explicit BIT_0/BIT_1x7 + // sequence, as opposed to a loop) because RyuJIT does not inline methods containing + // a loop - leaving this as a `for` loop, even with AggressiveInlining, forces this + // call to remain a real call boundary, which makes `rs` address-exposed and prevents + // the JIT from keeping Range/Code in registers for the rest of the CodeFast loop. + uint m = 1; + var prob = firstProb; + + var child0 = m << 1; + var probChild0 = p[child0]; + var probChild1 = p[child0 + 1]; + var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + prob = (probChild0 & ~mask) | (probChild1 & mask); + + child0 = m << 1; + bit = DecodeBitFastCore(ref rs, p + m, prob, out mask); + m = child0 + bit; + + return (byte)m; + } + + // Shared plain-literal-tree tail used once a matched-literal decode diverges from + // matchByte: from that point on there's no more matched-zone offset, so this is the same + // decode as LiteralDecodeNormalFast's inner step, just resuming mid-tree from `symbol`. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe byte LiteralDecodeTailFast(ref FastRangeState rs, ushort* p, uint symbol) + { + while (symbol < 0x100) + { + var bit = DecodeBitFastCore(ref rs, p + symbol, p[symbol], out _); + symbol = (symbol << 1) | bit; + } + return (byte)symbol; + } + + // baseP/firstIndex/firstProb are precomputed by the caller for the same reason as in + // LiteralDecodeNormalFast above (pos/prevByte/matchByte are all already known before the + // IsMatch decode resolves). + private static unsafe byte LiteralDecodeWithMatchByteFast( + ref FastRangeState rs, + ushort* baseP, + byte matchByte, + uint firstIndex, + uint firstProb + ) + { + var p = baseP; + + // Same one-level-ahead prefetch as BitTreeDecodeFast, adapted for the matched-literal + // zone: matchByte's bits are all known upfront (it comes from the dictionary, not from + // decoding), so the *next* iteration's matched-zone index can be computed from it + // immediately, independent of this iteration's decoded bit. + uint symbol = 1; + var matchBit = (uint)(matchByte >> 7) & 1; + matchByte <<= 1; + var index = (int)firstIndex; + var prob = firstProb; + + for (var i = 0; i < 8; i++) + { + var nextMatchBit = (uint)(matchByte >> 7) & 1; + matchByte <<= 1; + var hasNext = i < 7; + + var child0 = (uint)((1 + nextMatchBit) << 8) + (symbol << 1); + uint probChild0 = 0, + probChild1 = 0; + if (hasNext) + { + probChild0 = p[child0]; + probChild1 = p[child0 + 1]; + } + + var bit = DecodeBitFastCore(ref rs, p + index, prob, out var mask); + symbol = (symbol << 1) | bit; + + if (matchBit != bit) + { + return LiteralDecodeTailFast(ref rs, p, symbol); + } + + matchBit = nextMatchBit; + if (hasNext) + { + index = (int)(child0 + bit); + prob = (probChild0 & ~mask) | (probChild1 & mask); + } + } + return (byte)symbol; + } +} +#endif diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index 02fd16d7..60ac6a00 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -331,6 +331,12 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable _outWindow = null; } +#if !LEGACY_DOTNET + // On modern .NET targets, the hot decode loop uses the unsafe flat-array fast path + // (see LzmaDecoder.Fast.cs) instead of the per-array, method-call based loop below. + internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) => + CodeFast(dictionarySize, outWindow, rangeDecoder); +#else internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) { var dictionarySizeCheck = Math.Max(dictionarySize, 1); @@ -456,6 +462,7 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable } return false; } +#endif public void SetDecoderProperties(byte[] properties) => SetDecoderProperties(properties.AsSpan()); @@ -477,6 +484,10 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable SetLiteralProperties(lp, lc); SetPosBitsProperties(pb); Init(); +#if !LEGACY_DOTNET + CreateFastModel(lp, lc); + InitFastModel(); +#endif if (properties.Length >= 5) { _dictionarySize = 0; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index c89fb923..1c314705 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -175,6 +175,11 @@ public partial class LzmaStream CancellationToken cancellationToken ) { + if (count > 0) + { + EnsureReadMode(ReadMode.Asynchronous); + } + if (_endReached) { return 0; @@ -289,6 +294,11 @@ public partial class LzmaStream CancellationToken cancellationToken = default ) { + if (!buffer.IsEmpty) + { + EnsureReadMode(ReadMode.Asynchronous); + } + if (_endReached) { return 0; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index f45b33a3..832c0633 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -36,6 +36,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable private readonly Encoder? _encoder; private byte[]? _asyncHeaderBuffer; private bool _isDisposed; + private ReadMode _readMode; + + private enum ReadMode + { + None, + Synchronous, + Asynchronous, + } private LzmaStream( byte[] properties, @@ -131,6 +139,13 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } lzma._rangeDecoder.Init(inputStream); +#if !LEGACY_DOTNET + // Bound the fast buffered reader to the known compressed size (when available) so + // it never reads past this entry's data even on unbounded/shared streams. When the + // size is unknown (e.g. Zip data-descriptor entries), RangeCoder.Decoder falls back + // to conservative per-byte reads on streams that cannot report a safe Length. + lzma._rangeDecoder.SetFastLimit(lzma._rangeDecoderLimit); +#endif } else { @@ -242,6 +257,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable public override int Read(byte[] buffer, int offset, int count) { + if (count > 0) + { + EnsureReadMode(ReadMode.Synchronous); + } + if (_endReached) { return 0; @@ -337,6 +357,8 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable public override int ReadByte() { + EnsureReadMode(ReadMode.Synchronous); + if (_endReached) { return -1; @@ -416,6 +438,22 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable return value; } + private void EnsureReadMode(ReadMode readMode) + { + if (_readMode == ReadMode.None) + { + _readMode = readMode; + return; + } + + if (_readMode != readMode) + { + throw new InvalidOperationException( + "A LzmaStream cannot mix synchronous and asynchronous reads." + ); + } + } + private void DecodeChunkHeader() { var control = _inputStream!.ReadByte(); @@ -474,6 +512,13 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } _rangeDecoder.Init(_inputStream); +#if !LEGACY_DOTNET + // LZMA2 chunks share one underlying stream with the raw chunk-header bytes read + // above/below, so the buffered fast-read path (see RangeCoder.Decoder's fast + // buffer) must never physically read past this chunk's compressed size, or it + // would desynchronize the stream position for the next chunk header. + _rangeDecoder.SetFastLimit(_rangeDecoderLimit); +#endif } else if (control > 0x02) { diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs index 147146c4..141b1dbd 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs @@ -1,7 +1,11 @@ #nullable disable +using System; using System.IO; using System.Runtime.CompilerServices; +#if !LEGACY_DOTNET +using System.Buffers; +#endif namespace SharpCompress.Compressors.LZMA.RangeCoder; @@ -89,6 +93,33 @@ internal partial class Decoder public long _total; private byte[] _singleByteBuffer; +#if !LEGACY_DOTNET + // Upper bound (in terms of _total) that the fast buffered reader is allowed to physically + // read up to. -1 means unbounded. This matters for formats like LZMA2 where multiple + // independent chunks share one underlying stream: chunk headers are read directly from that + // stream between decode sessions, so the fast path must not read past the end of the + // current chunk's compressed data, or it would desynchronize the stream position for the + // next chunk header read. Set via SetFastLimit once the caller knows the chunk/stream size. + private long _fastLimit = -1; + + // Whether it is safe to bulk-read ahead of the decoder even without a known _fastLimit. + // This is only true for streams that are guaranteed to self-clamp Read() to their own + // logical end and never return bytes that belong to something else - e.g. 7Zip's + // per-folder BufferedSubStream, which limits every Read() to its own remaining pack size. + // It is false for everything else, notably a shared streaming Zip reader stream with an + // unknown compressed size (data-descriptor entries): that stream keeps handing out bytes + // past the logical end of this LZMA stream (the next entry's header, etc.) with no self + // clamping and no way to give unread bytes back, so bulk-buffering there would + // desynchronize the stream. Note a stream reporting a queryable Length is NOT a reliable + // signal here: some wrapper streams (e.g. SharpCompressStream's ring-buffer mode used for + // over-read recording on non-seekable Zip streams) expose a Length without actually + // bounding Read() to the current logical stream's end. In the unsafe case we fall back to + // reading exactly one byte at a time, matching the legacy per-byte ReadByte() behavior. + private bool _fastBufferSafeUnbounded; + + public void SetFastLimit(long limit) => _fastLimit = limit; +#endif + public void Init(Stream stream) { _stream = stream; @@ -100,11 +131,105 @@ internal partial class Decoder _code = (_code << 8) | (byte)_stream.ReadByte(); } _total = 5; +#if !LEGACY_DOTNET + _fastLimit = -1; + _fastBufferPos = 0; + _fastBufferLen = 0; + _fastEndOfStream = false; + _fastBufferSafeUnbounded = stream is SharpCompress.IO.BufferedSubStream; +#endif } - public void ReleaseStream() => + public void ReleaseStream() + { +#if !LEGACY_DOTNET + ReleaseFastBuffer(); +#endif // Stream.ReleaseStream(); _stream = null; + } + +#if !LEGACY_DOTNET + // Buffered input used only by the unsafe fast LZMA decode path (see LzmaDecoder.Fast.cs). + // Avoids issuing a virtual Stream.ReadByte() call per consumed byte, which otherwise + // dominates decode time (millions of calls for a typical archive). + private const int FastBufferSize = 1 << 16; + private byte[] _fastBuffer; + private int _fastBufferPos; + private int _fastBufferLen; + private bool _fastEndOfStream; + + // The fast decode loop (LzmaDecoder.Fast.cs) pins this array and keeps its own local + // copies of position/length/consumed-byte-count for the duration of a decode call, only + // syncing back through these accessors when the local buffer is exhausted (rare) or when + // the decode call ends. This avoids reloading these fields from this object on every bit. + internal byte[] FastBufferArray => _fastBuffer ??= ArrayPool.Shared.Rent(FastBufferSize); + + internal int FastBufferPos + { + get => _fastBufferPos; + set => _fastBufferPos = value; + } + + internal int FastBufferLen => _fastBufferLen; + + internal void AddTotal(long consumed) => _total += consumed; + + internal void RefillFast() => FillFastBuffer(); + + private void FillFastBuffer() + { + _fastBuffer ??= ArrayPool.Shared.Rent(FastBufferSize); + if (_fastEndOfStream) + { + // Match legacy (byte)_stream.ReadByte() behavior: once the stream is + // exhausted, keep yielding 0xFF instead of throwing or re-reading. + _fastBufferPos = 0; + _fastBufferLen = 1; + _fastBuffer[0] = 0xFF; + return; + } + var requestSize = _fastBuffer.Length; + if (_fastLimit >= 0) + { + var remaining = _fastLimit - _total; + requestSize = remaining <= 0 ? 1 : (int)Math.Min(requestSize, remaining); + } + else if (!_fastBufferSafeUnbounded) + { + // No known limit and the underlying stream cannot report a bounded Length (e.g. a + // shared, forward-only stream such as a streaming Zip reader). Reading ahead here + // could consume bytes that belong to whatever comes after this logical LZMA stream + // (the next Zip entry's header, etc.) with no way to give them back. Fall back to + // requesting exactly one byte at a time so we never consume more than the decoder + // actually needs, matching the legacy per-byte ReadByte() behavior. + requestSize = 1; + } + var read = _stream.Read(_fastBuffer, 0, requestSize); + if (read <= 0) + { + _fastEndOfStream = true; + _fastBufferPos = 0; + _fastBufferLen = 1; + _fastBuffer[0] = 0xFF; + return; + } + _fastBufferPos = 0; + _fastBufferLen = read; + } + + private void ReleaseFastBuffer() + { + if (_fastBuffer is not null) + { + ArrayPool.Shared.Return(_fastBuffer); + _fastBuffer = null; + } + _fastBufferPos = 0; + _fastBufferLen = 0; + _fastEndOfStream = false; + } +#endif public void Normalize() { diff --git a/src/SharpCompress/Compressors/Rar/RarCRC.cs b/src/SharpCompress/Compressors/Rar/RarCRC.cs index 7e6fc6cc..ee122d21 100644 --- a/src/SharpCompress/Compressors/Rar/RarCRC.cs +++ b/src/SharpCompress/Compressors/Rar/RarCRC.cs @@ -1,10 +1,16 @@ using System; +using SharpCompress.Crypto; namespace SharpCompress.Compressors.Rar; internal static class RarCRC { - private static readonly uint[] crcTab; + // Reuse Crc32Stream's cached slice-by-16 table (same polynomial) instead of a separate + // byte-at-a-time table, so bulk CRC checks during Rar extraction get the same throughput + // as Zip/GZip/7Zip/LZip CRC32 validation instead of a much slower one-byte-per-iteration loop. + private static readonly uint[] crcTab = Crc32Stream.InitializeTable( + Crc32Stream.DEFAULT_POLYNOMIAL + ); public static uint CheckCrc(uint startCrc, byte b) => (crcTab[((int)startCrc ^ b) & 0xff] ^ (startCrc >> 8)); @@ -12,35 +18,6 @@ internal static class RarCRC public static uint CheckCrc(uint startCrc, ReadOnlySpan data, int offset, int count) { var size = Math.Min(data.Length - offset, count); - - for (var i = 0; i < size; i++) - { - startCrc = (crcTab[((int)startCrc ^ data[offset + i]) & 0xff] ^ (startCrc >> 8)); - } - return (startCrc); - } - - static RarCRC() - { - { - crcTab = new uint[256]; - for (uint i = 0; i < 256; i++) - { - var c = i; - for (var j = 0; j < 8; j++) - { - if ((c & 1) != 0) - { - c >>= 1; - c ^= 0xEDB88320; - } - else - { - c >>= 1; - } - } - crcTab[i] = c; - } - } + return Crc32Stream.CalculateCrc(crcTab, startCrc, data.Slice(offset, size)); } } diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs index 2aedc9a6..0042af7a 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; @@ -31,16 +30,16 @@ internal partial class RarCrcStream : RarStream .ConfigureAwait(false); if (result != 0) { - currentCrc = RarCRC.CheckCrc(currentCrc, buffer, offset, result); + _currentCrc = RarCRC.CheckCrc(_currentCrc, buffer, offset, result); } else if ( - !disableCRC - && GetCrc() != BitConverter.ToUInt32(readStream.NotNull().CurrentCrc.NotNull(), 0) + !_disableCrc + && GetCrc() != BitConverter.ToUInt32(_readStream.NotNull().CurrentCrc.NotNull(), 0) && count != 0 ) { // NOTE: we use the last FileHeader in a multipart volume to check CRC - throw new InvalidFormatException("file crc mismatch"); + throw new InvalidFormatException("file crc mismatch: " + _key); } return result; @@ -56,16 +55,16 @@ internal partial class RarCrcStream : RarStream var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); if (result != 0) { - currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span, 0, result); + _currentCrc = RarCRC.CheckCrc(_currentCrc, buffer.Span, 0, result); } else if ( - !disableCRC - && GetCrc() != BitConverter.ToUInt32(readStream.NotNull().CurrentCrc.NotNull(), 0) + !_disableCrc + && GetCrc() != BitConverter.ToUInt32(_readStream.NotNull().CurrentCrc.NotNull(), 0) && buffer.Length != 0 ) { // NOTE: we use the last FileHeader in a multipart volume to check CRC - throw new InvalidFormatException("file crc mismatch"); + throw new InvalidFormatException("file crc mismatch: " + _key); } return result; diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index a9025e19..233b05be 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -1,7 +1,4 @@ using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; @@ -9,9 +6,10 @@ namespace SharpCompress.Compressors.Rar; internal partial class RarCrcStream : RarStream { - private readonly MultiVolumeReadOnlyStreamBase readStream; - private uint currentCrc; - private readonly bool disableCRC; + private readonly string? _key; + private readonly MultiVolumeReadOnlyStreamBase _readStream; + private uint _currentCrc; + private readonly bool _disableCrc; private RarCrcStream( IRarUnpack unpack, @@ -20,8 +18,9 @@ internal partial class RarCrcStream : RarStream ) : base(unpack, fileHeader, readStream) { - this.readStream = readStream; - disableCRC = fileHeader.IsEncrypted; + this._readStream = readStream; + _key = fileHeader.FileName; + _disableCrc = fileHeader.IsEncrypted; ResetCrc(); } @@ -36,31 +35,25 @@ internal partial class RarCrcStream : RarStream } // Async methods moved to RarCrcStream.Async.cs + public uint GetCrc() => ~_currentCrc; - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public uint GetCrc() => ~currentCrc; - - public void ResetCrc() => currentCrc = 0xffffffff; + public void ResetCrc() => _currentCrc = 0xffffffff; public override int Read(byte[] buffer, int offset, int count) { var result = base.Read(buffer, offset, count); if (result != 0) { - currentCrc = RarCRC.CheckCrc(currentCrc, buffer, offset, result); + _currentCrc = RarCRC.CheckCrc(_currentCrc, buffer, offset, result); } else if ( - !disableCRC - && GetCrc() != BitConverter.ToUInt32(readStream.NotNull().CurrentCrc.NotNull(), 0) + !_disableCrc + && GetCrc() != BitConverter.ToUInt32(_readStream.NotNull().CurrentCrc.NotNull(), 0) && count != 0 ) { // NOTE: we use the last FileHeader in a multipart volume to check CRC - throw new InvalidFormatException("file crc mismatch"); + throw new InvalidFormatException("file crc mismatch: " + _key); } return result; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_async.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_async.cs index cabce8cf..fbb2d607 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_async.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_async.cs @@ -394,28 +394,34 @@ internal partial class Unpack { await UnpIO_UnpWriteAsync(OutMem, 0, BlockLength, cancellationToken) .ConfigureAwait(false); - WrittenFileSize += BlockLength; } + UnpSomeRead = true; + WrittenFileSize += BlockLength; WrittenBorder = BlockEnd; WriteSizeLeft = (UnpPtr - WrittenBorder) & MaxWinMask; } } else { - NotAllFiltersProcessed = true; + // Current filter intersects the window write border, so we adjust + // the window border to process this filter next time, not now. + WrPtr = WrittenBorder; + + // Since Filter start position can only increase, we quit processing + // all following filters for this data block and reset 'NextWindow' + // flag for them. for (var J = I; J < Filters.Count; J++) { var fltj = Filters[J]; - if ( - fltj.Type != FILTER_NONE - && fltj.NextWindow == false - && ((fltj.BlockStart - WrPtr) & MaxWinMask) < FullWriteSize - ) + if (fltj.Type != FILTER_NONE) { - fltj.NextWindow = true; + fltj.NextWindow = false; } } + + // Do not write data left after current filter now. + NotAllFiltersProcessed = true; break; } } diff --git a/src/SharpCompress/Crypto/Crc32Stream.cs b/src/SharpCompress/Crypto/Crc32Stream.cs index 67dcb913..8d72f23f 100644 --- a/src/SharpCompress/Crypto/Crc32Stream.cs +++ b/src/SharpCompress/Crypto/Crc32Stream.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers.Binary; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -119,6 +120,14 @@ public sealed class Crc32Stream : Stream public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer) => ~CalculateCrc(InitializeTable(polynomial), seed, buffer); + // Number of "slice-by-N" tables generated per polynomial (see InitializeTable). 16 gives a + // good throughput/table-size tradeoff (16 * 256 * 4 bytes = 16KB per polynomial, generated + // once and cached for the default polynomial) - a well-known technique (zlib-ng, Chromium, + // etc.) for speeding up table-driven CRC32 by processing 16 bytes/iteration through 16 + // independent (parallelizable) table lookups instead of 1 byte/iteration through a single + // lookup on the CPU's dependency chain. + private const int SliceTableCount = 16; + internal static uint[] InitializeTable(uint polynomial) { if (polynomial == DEFAULT_POLYNOMIAL && _defaultTable != null) @@ -126,38 +135,80 @@ public sealed class Crc32Stream : Stream return _defaultTable; } - var createTable = new uint[256]; + // table[0..255] is the ordinary byte-wise CRC table (unchanged layout, so existing + // single-byte callers - CalculateCrc(table, crc, byte) - keep working unmodified). + // table[256*k .. 256*k+255] for k=1..SliceTableCount-1 are the derived "slice" tables: + // table[k][i] = (table[k-1][i] >> 8) ^ table[0][table[k-1][i] & 0xFF], i.e. each table + // folds in the effect of a byte that is k positions further back in the stream than + // the byte table[0] would have processed - letting CalculateCrc(span) below combine 4 + // bytes' worth of independent lookups per 32-bit word with no cross-lookup dependency. + var table = new uint[256 * SliceTableCount]; for (var i = 0; i < 256; i++) { var entry = (uint)i; for (var j = 0; j < 8; j++) { - if ((entry & 1) == 1) - { - entry = (entry >> 1) ^ polynomial; - } - else - { - entry >>= 1; - } + entry = (entry & 1) == 1 ? (entry >> 1) ^ polynomial : entry >> 1; } + table[i] = entry; + } - createTable[i] = entry; + for (var k = 1; k < SliceTableCount; k++) + { + var prevBase = (k - 1) * 256; + var curBase = k * 256; + for (var i = 0; i < 256; i++) + { + var prev = table[prevBase + i]; + table[curBase + i] = (prev >> 8) ^ table[prev & 0xFF]; + } } if (polynomial == DEFAULT_POLYNOMIAL) { - _defaultTable = createTable; + _defaultTable = table; } - return createTable; + return table; } internal static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan buffer) { unchecked { - for (var i = 0; i < buffer.Length; i++) + var i = 0; + if (table.Length >= 256 * SliceTableCount) + { + while (buffer.Length - i >= 16) + { + var word0 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i)) ^ crc; + var word1 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 4)); + var word2 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 8)); + var word3 = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(i + 12)); + + crc = + table[15 * 256 + (byte)word0] + ^ table[14 * 256 + (byte)(word0 >> 8)] + ^ table[13 * 256 + (byte)(word0 >> 16)] + ^ table[12 * 256 + (word0 >> 24)] + ^ table[11 * 256 + (byte)word1] + ^ table[10 * 256 + (byte)(word1 >> 8)] + ^ table[9 * 256 + (byte)(word1 >> 16)] + ^ table[8 * 256 + (word1 >> 24)] + ^ table[7 * 256 + (byte)word2] + ^ table[6 * 256 + (byte)(word2 >> 8)] + ^ table[5 * 256 + (byte)(word2 >> 16)] + ^ table[4 * 256 + (word2 >> 24)] + ^ table[3 * 256 + (byte)word3] + ^ table[2 * 256 + (byte)(word3 >> 8)] + ^ table[1 * 256 + (byte)(word3 >> 16)] + ^ table[word3 >> 24]; + + i += 16; + } + } + + for (; i < buffer.Length; i++) { crc = CalculateCrc(table, crc, buffer[i]); } diff --git a/src/SharpCompress/Detection/ArchiveAccessMode.cs b/src/SharpCompress/Detection/ArchiveAccessMode.cs new file mode 100644 index 00000000..8826f2d8 --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveAccessMode.cs @@ -0,0 +1,25 @@ +using System; + +namespace SharpCompress.Detection; + +/// +/// Specifies the APIs available for an archive format. +/// +[Flags] +public enum ArchiveAccessMode +{ + /// + /// No archive access API is available. + /// + None = 0, + + /// + /// The seekable API is available. + /// + Archive = 1, + + /// + /// The forward-only API is available. + /// + Reader = 2, +} diff --git a/src/SharpCompress/Detection/ArchiveDetection.cs b/src/SharpCompress/Detection/ArchiveDetection.cs new file mode 100644 index 00000000..c01cf4b5 --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveDetection.cs @@ -0,0 +1,43 @@ +using SharpCompress.Common; + +namespace SharpCompress.Detection; + +/// +/// Identifies an archive format without enumerating its entries. +/// +public sealed class ArchiveDetection +{ + internal ArchiveDetection( + ArchiveType? containerType, + string formatName, + CompressionType? outerCompressionType, + ArchiveAccessMode supportedApis + ) + { + ContainerType = containerType; + FormatName = formatName; + OuterCompressionType = outerCompressionType; + SupportedApis = supportedApis; + } + + /// + /// Gets the logical archive container type, when it is a built-in SharpCompress type. + /// + public ArchiveType? ContainerType { get; } + + /// + /// Gets the detected format name. + /// + public string FormatName { get; } + + /// + /// Gets the compression wrapper around , if present. + /// For example, a tar.xz archive has a TAR container and XZ outer compression. + /// + public CompressionType? OuterCompressionType { get; } + + /// + /// Gets the APIs supported by the detected format. + /// + public ArchiveAccessMode SupportedApis { get; } +} diff --git a/src/SharpCompress/Detection/ArchiveEncryptionScope.cs b/src/SharpCompress/Detection/ArchiveEncryptionScope.cs new file mode 100644 index 00000000..2e3afe8c --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveEncryptionScope.cs @@ -0,0 +1,25 @@ +using System; + +namespace SharpCompress.Detection; + +/// +/// Identifies which archive data is encrypted. +/// +[Flags] +public enum ArchiveEncryptionScope +{ + /// + /// No encryption is present. + /// + None = 0, + + /// + /// One or more entry payloads are encrypted. + /// + EntryData = 1, + + /// + /// Archive headers are encrypted. + /// + Headers = 2, +} diff --git a/src/SharpCompress/Detection/ArchiveInformation.cs b/src/SharpCompress/Detection/ArchiveInformation.cs new file mode 100644 index 00000000..fac49816 --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveInformation.cs @@ -0,0 +1,145 @@ +namespace SharpCompress.Detection; + +/// +/// Contains metadata collected by fully inspecting an archive. +/// +public sealed class ArchiveInformation +{ + internal ArchiveInformation( + ArchiveDetection detection, + ArchiveInformationStatus status, + ArchiveInformationLimitations limitations, + string? formatVersion, + long? entryCount, + long? entriesWithUnknownSizeCount, + long? physicalSize, + long? compressedPayloadSize, + long? uncompressedPayloadSize, + bool? isSolid, + long? solidStreamCount, + ArchiveEncryptionScope? encryption, + bool? isEncrypted, + bool? isMultiVolume, + int? physicalPartCount, + int? logicalVolumeCount, + bool? isComplete, + string? comment, + ZipArchiveInformation? zip + ) + { + Detection = detection; + Status = status; + Limitations = limitations; + FormatVersion = formatVersion; + EntryCount = entryCount; + EntriesWithUnknownSizeCount = entriesWithUnknownSizeCount; + PhysicalSize = physicalSize; + CompressedPayloadSize = compressedPayloadSize; + UncompressedPayloadSize = uncompressedPayloadSize; + IsSolid = isSolid; + SolidStreamCount = solidStreamCount; + Encryption = encryption; + IsEncrypted = isEncrypted; + IsMultiVolume = isMultiVolume; + PhysicalPartCount = physicalPartCount; + LogicalVolumeCount = logicalVolumeCount; + IsComplete = isComplete; + Comment = comment; + Zip = zip; + } + + /// + /// Gets the format and API capabilities identified before inspection. + /// + public ArchiveDetection Detection { get; } + + /// + /// Gets whether metadata collection completed. + /// + public ArchiveInformationStatus Status { get; } + + /// + /// Gets conditions that prevented complete metadata collection. + /// + public ArchiveInformationLimitations Limitations { get; } + + /// + /// Gets the format version when exposed by the archive format. + /// + public string? FormatVersion { get; } + + /// + /// Gets the number of entries, excluding format control records. + /// + public long? EntryCount { get; } + + /// + /// Gets the number of entries for which a forward-only reader cannot know the uncompressed size before reading entry data. + /// + public long? EntriesWithUnknownSizeCount { get; } + + /// + /// Gets the number of bytes in the supplied source parts, when known. + /// + public long? PhysicalSize { get; } + + /// + /// Gets the aggregate stored payload size, when the format exposes it reliably. + /// + public long? CompressedPayloadSize { get; } + + /// + /// Gets the aggregate uncompressed entry size, when the format exposes it reliably. + /// + public long? UncompressedPayloadSize { get; } + + /// + /// Gets whether entries share compression state. + /// + public bool? IsSolid { get; } + + /// + /// Gets the number of independent shared compression streams, when known. + /// + public long? SolidStreamCount { get; } + + /// + /// Gets the scopes protected by encryption, when known. + /// + public ArchiveEncryptionScope? Encryption { get; } + + /// + /// Gets whether archive headers or entry data are encrypted, when known. + /// + public bool? IsEncrypted { get; } + + /// + /// Gets whether the archive is part of a multi-volume set, when known. + /// + public bool? IsMultiVolume { get; } + + /// + /// Gets the number of source parts supplied for inspection. + /// + public int? PhysicalPartCount { get; } + + /// + /// Gets the number of logical volumes exposed by the archive, when known. + /// + public int? LogicalVolumeCount { get; } + + /// + /// Gets whether all discovered entries are complete, when known. + /// + public bool? IsComplete { get; } + + /// + /// Gets the archive comment, when supported by the format. + /// + public string? Comment { get; } + + /// + /// Gets ZIP-specific metadata when the detected container is ZIP. + /// + public ZipArchiveInformation? Zip { get; } +} diff --git a/src/SharpCompress/Detection/ArchiveInformationLimitations.cs b/src/SharpCompress/Detection/ArchiveInformationLimitations.cs new file mode 100644 index 00000000..592f6ed6 --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveInformationLimitations.cs @@ -0,0 +1,30 @@ +using System; + +namespace SharpCompress.Detection; + +/// +/// Identifies conditions that prevented complete archive metadata inspection. +/// +[Flags] +public enum ArchiveInformationLimitations +{ + /// + /// No known inspection limitations apply. + /// + None = 0, + + /// + /// Archive headers are encrypted and no password was supplied. + /// + EncryptedHeaders = 1, + + /// + /// One or more archive volumes are unavailable. + /// + MissingVolumes = 2, + + /// + /// The format does not expose a requested metadata value. + /// + UnavailableMetadata = 4, +} diff --git a/src/SharpCompress/Detection/ArchiveInformationStatus.cs b/src/SharpCompress/Detection/ArchiveInformationStatus.cs new file mode 100644 index 00000000..5d37f534 --- /dev/null +++ b/src/SharpCompress/Detection/ArchiveInformationStatus.cs @@ -0,0 +1,17 @@ +namespace SharpCompress.Detection; + +/// +/// Describes whether archive metadata could be collected completely. +/// +public enum ArchiveInformationStatus +{ + /// + /// All metadata supported by the format was collected. + /// + Complete, + + /// + /// Some metadata could not be collected. See . + /// + Partial, +} diff --git a/src/SharpCompress/Detection/ZipArchiveInformation.cs b/src/SharpCompress/Detection/ZipArchiveInformation.cs new file mode 100644 index 00000000..63e46e1b --- /dev/null +++ b/src/SharpCompress/Detection/ZipArchiveInformation.cs @@ -0,0 +1,16 @@ +namespace SharpCompress.Detection; + +/// +/// Contains ZIP-specific metadata. +/// +public sealed class ZipArchiveInformation +{ + internal ZipArchiveInformation(bool hasEntriesWithDeferredSizes) => + HasEntriesWithDeferredSizes = hasEntriesWithDeferredSizes; + + /// + /// Gets whether any entry sizes are available only after reading the entry data. + /// When , a forward-only reader can initially report zero sizes for those entries. + /// + public bool HasEntriesWithDeferredSizes { get; } +} diff --git a/src/SharpCompress/Factories/GZipFactory.cs b/src/SharpCompress/Factories/GZipFactory.cs index bda6ef1f..7dfdd313 100644 --- a/src/SharpCompress/Factories/GZipFactory.cs +++ b/src/SharpCompress/Factories/GZipFactory.cs @@ -151,13 +151,16 @@ public class GZipFactory CompressionContext.FromStream(sharpCompressStream).WithReaderOptions(options) ) ); - if (TarArchive.IsTarFile(testStream)) + var isTarArchive = TarArchive.IsTarFile(testStream); + + // The TAR probe can consume arbitrary compressed input before it rejects a stream. + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - sharpCompressStream.StopRecording(); reader = new TarReader(sharpCompressStream, options, CompressionType.GZip); return true; } - sharpCompressStream.StopRecording(); reader = OpenReader(sharpCompressStream, options); return true; } @@ -182,15 +185,27 @@ public class GZipFactory } sharpCompressStream.Rewind(); - var tarReader = await new TarFactory() - .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken) + using var testStream = SharpCompressStream.CreateNonDisposing( + await options + .Providers.CreateDecompressStreamAsync( + CompressionType.GZip, + SharpCompressStream.CreateNonDisposing(sharpCompressStream), + CompressionContext.FromStream(sharpCompressStream).WithReaderOptions(options), + cancellationToken + ) + .ConfigureAwait(false) + ); + var isTarArchive = await TarArchive + .IsTarFileAsync(testStream, cancellationToken) .ConfigureAwait(false); - if (tarReader is not null) + + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - return tarReader; + return new TarReader(sharpCompressStream, options, CompressionType.GZip); } - sharpCompressStream.StopRecording(); return await OpenAsyncReader(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Factories/LzwFactory.cs b/src/SharpCompress/Factories/LzwFactory.cs index bd5569d1..098ec002 100644 --- a/src/SharpCompress/Factories/LzwFactory.cs +++ b/src/SharpCompress/Factories/LzwFactory.cs @@ -65,14 +65,17 @@ public class LzwFactory : Factory, IReaderFactory ) ) { - if (TarArchive.IsTarFile(testStream)) + var isTarArchive = TarArchive.IsTarFile(testStream); + + // The TAR probe can consume arbitrary compressed input before it rejects a stream. + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - sharpCompressStream.StopRecording(); reader = new TarReader(sharpCompressStream, options, CompressionType.Lzw); return true; } } - sharpCompressStream.StopRecording(); reader = OpenReader(sharpCompressStream, options); return true; } @@ -97,15 +100,26 @@ public class LzwFactory : Factory, IReaderFactory } sharpCompressStream.Rewind(); - var tarReader = await new TarFactory() - .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken) + using var testStream = SharpCompressStream.CreateNonDisposing( + await options + .Providers.CreateDecompressStreamAsync( + CompressionType.Lzw, + SharpCompressStream.CreateNonDisposing(sharpCompressStream), + cancellationToken + ) + .ConfigureAwait(false) + ); + var isTarArchive = await TarArchive + .IsTarFileAsync(testStream, cancellationToken) .ConfigureAwait(false); - if (tarReader is not null) + + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - return tarReader; + return new TarReader(sharpCompressStream, options, CompressionType.Lzw); } - sharpCompressStream.StopRecording(); return await OpenAsyncReader(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Factories/TarFactory.cs b/src/SharpCompress/Factories/TarFactory.cs index 7c88b1d6..38dd7ab9 100644 --- a/src/SharpCompress/Factories/TarFactory.cs +++ b/src/SharpCompress/Factories/TarFactory.cs @@ -296,11 +296,53 @@ public class TarFactory #region IReaderFactory + /// + /// Checks whether is backed by a genuinely seekable stream. + /// Note: always reports for + /// non-passthrough instances (it simulates seeking via its ring buffer even over a + /// non-seekable underlying stream), so it must be unwrapped to check the real stream. + /// + private static bool IsGenuinelySeekable(Stream stream) + { + while (stream is SharpCompressStream scs) + { + stream = scs.BaseStream(); + } + return stream.CanSeek; + } + + /// + /// Creates a new wrapper around with + /// its own independent recording/rewind scope, isolated from any recording session the caller may + /// already have in progress on itself. + /// + /// + /// When the stream is genuinely seekable, a is used: + /// it delegates recording/rewinding to the underlying stream's native and + /// never allocates a rewind ring buffer. For non-seekable streams, a ring-buffered + /// is required to support rewinding during format detection. + /// + private static SharpCompressStream CreateNestedRecordingStream(Stream stream) + { + if (!IsGenuinelySeekable(stream)) + { + return new SharpCompressStream(stream); + } + + // Unwrap buffered SharpCompressStream wrappers so we use the real stream's native Seek. + while (stream is SharpCompressStream scs && !scs.IsPassthrough) + { + stream = scs.BaseStream(); + } + + return new SeekableSharpCompressStream(stream, leaveStreamOpen: false); + } + /// public IReader OpenReader(Stream stream, ReaderOptions? options) { options ??= ReaderOptions.ForExternalStream; - var sharpCompressStream = new SharpCompressStream(stream); + var sharpCompressStream = CreateNestedRecordingStream(stream); sharpCompressStream.StartRecording(TarWrapper.MaximumRewindBufferSize); foreach (var wrapper in TarWrapper.Wrappers) { @@ -315,6 +357,7 @@ public class TarFactory ); if (TarArchive.IsTarFile(decompressedStream)) { + sharpCompressStream.Rewind(); sharpCompressStream.StopRecording(); return new TarReader(sharpCompressStream, options, wrapper.CompressionType); } diff --git a/src/SharpCompress/IO/ArchiveOffsetStream.cs b/src/SharpCompress/IO/ArchiveOffsetStream.cs new file mode 100644 index 00000000..9c7bc03e --- /dev/null +++ b/src/SharpCompress/IO/ArchiveOffsetStream.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +/// +/// Exposes the remainder of a seekable stream as an archive that begins at position zero. +/// +internal sealed class ArchiveOffsetStream : Stream +{ + private readonly Stream stream; + private readonly long origin; + + public ArchiveOffsetStream(Stream stream) + { + this.stream = stream; + origin = stream.Position; + } + + public override bool CanRead => stream.CanRead; + public override bool CanSeek => stream.CanSeek; + public override bool CanWrite => false; + public override long Length => stream.Length - origin; + + public override long Position + { + get => stream.Position - origin; + set => stream.Position = origin + value; + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => + stream.Read(buffer, offset, count); + +#if !LEGACY_DOTNET + public override int Read(Span buffer) => stream.Read(buffer); +#endif + + public override int ReadByte() => stream.ReadByte(); + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) => stream.ReadAsync(buffer, offset, count, cancellationToken); + +#if !NET48 && !NETSTANDARD2_0 + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) => stream.ReadAsync(buffer, cancellationToken); +#endif + + public override long Seek(long offset, SeekOrigin seekOrigin) + { + var position = seekOrigin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => Position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(seekOrigin)), + }; + + if (position < 0) + { + throw new IOException("Attempted to seek before the archive origin."); + } + + stream.Seek(origin + position, SeekOrigin.Begin); + return position; + } + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + // The caller owns the source stream. + base.Dispose(disposing); + } +} diff --git a/src/SharpCompress/IO/AsyncDisposableStream.cs b/src/SharpCompress/IO/AsyncDisposableStream.cs new file mode 100644 index 00000000..4c7fafbe --- /dev/null +++ b/src/SharpCompress/IO/AsyncDisposableStream.cs @@ -0,0 +1,36 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +/// +/// A that is guaranteed to be asynchronously disposable on every target framework. +/// +/// +/// +/// On .NET Framework 4.8 and .NET Standard 2.0, has no DisposeAsync. +/// Microsoft.Bcl.AsyncInterfaces supplies the interface on those +/// targets but cannot retrofit it onto the BCL's , and C# will not accept an +/// extension method for the pattern - await using requires a reachable instance +/// DisposeAsync. Deriving from this class instead of therefore makes a type +/// usable with await using uniformly, with no conditional compilation at the call site. +/// +/// +/// The fallback below is the same behaviour as the BCL's own default , +/// so a derived type may call await base.DisposeAsync() unconditionally on any target. +/// +/// +public abstract class AsyncDisposableStream : Stream +#if NO_STREAM_DISPOSEASYNC + , IAsyncDisposable +#endif +{ +#if NO_STREAM_DISPOSEASYNC + public virtual ValueTask DisposeAsync() + { + Dispose(); + return default; + } +#endif +} diff --git a/src/SharpCompress/IO/AsyncDisposeScope.cs b/src/SharpCompress/IO/AsyncDisposeScope.cs new file mode 100644 index 00000000..9d4538e4 --- /dev/null +++ b/src/SharpCompress/IO/AsyncDisposeScope.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +/// +/// Makes any resource usable with await using, disposing it asynchronously when the runtime type +/// supports it and synchronously otherwise. +/// +/// +/// Needed for locals whose static type is (or another type that +/// only sometimes has DisposeAsync), where await using cannot bind directly on +/// .NET Framework 4.8 / .NET Standard 2.0. Unlike a compile-time guard, this picks the asynchronous path +/// based on the runtime type, so a stream that really is asynchronously disposable is disposed that way on +/// every target framework. Prefer deriving from where the type is ours. +/// +internal readonly struct AsyncDisposeScope(IDisposable? resource) : IAsyncDisposable +{ + public ValueTask DisposeAsync() + { + if (resource is IAsyncDisposable asyncDisposable) + { + return asyncDisposable.DisposeAsync(); + } + + resource?.Dispose(); + return default; + } + + /// + /// Mirrors ConfiguredAsyncDisposable so await using can specify context capture without + /// boxing this struct through . + /// + public ConfiguredAsyncDisposeScope ConfigureAwait(bool continueOnCapturedContext) => + new(resource, continueOnCapturedContext); +} diff --git a/src/SharpCompress/IO/BufferedSubStream.Async.cs b/src/SharpCompress/IO/BufferedSubStream.Async.cs index 04c85b07..733cfc7e 100644 --- a/src/SharpCompress/IO/BufferedSubStream.Async.cs +++ b/src/SharpCompress/IO/BufferedSubStream.Async.cs @@ -36,19 +36,41 @@ internal partial class BufferedSubStream BytesLeftToRead -= _cacheLength; } - [Zomp.SyncMethodGenerator.CreateSyncVersion] - public override async Task ReadAsync( + // Fast, synchronous-completion path used when the requested bytes are already sitting in the + // in-memory cache. Avoids the async state-machine / Task allocation overhead incurred by every + // single-byte read the LZMA range decoder issues, without changing buffering/caching semantics. + public override Task ReadAsync( byte[] buffer, int offset, int count, CancellationToken cancellationToken ) { + cancellationToken.ThrowIfCancellationRequested(); + if (count > Length) { count = (int)Length; } + if (count > 0 && _cacheOffset < _cacheLength) + { + count = Math.Min(count, _cacheLength - _cacheOffset); + Buffer.BlockCopy(_cache!, _cacheOffset, buffer, offset, count); + _cacheOffset += count; + return Task.FromResult(count); + } + + return ReadSlowAsync(buffer, offset, count, cancellationToken); + } + + private async Task ReadSlowAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { if (count > 0) { if (_cacheOffset == _cacheLength) @@ -65,10 +87,34 @@ internal partial class BufferedSubStream } #if !LEGACY_DOTNET - public override async ValueTask ReadAsync( + public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default ) + { + cancellationToken.ThrowIfCancellationRequested(); + + var count = buffer.Length; + if (count > Length) + { + count = (int)Length; + } + + if (count > 0 && _cacheOffset < _cacheLength) + { + count = Math.Min(count, _cacheLength - _cacheOffset); + _cache!.AsSpan(_cacheOffset, count).CopyTo(buffer.Span); + _cacheOffset += count; + return new ValueTask(count); + } + + return ReadSlowAsync(buffer, cancellationToken); + } + + private async ValueTask ReadSlowAsync( + Memory buffer, + CancellationToken cancellationToken + ) { var count = buffer.Length; if (count > Length) diff --git a/src/SharpCompress/IO/BufferedSubStream.cs b/src/SharpCompress/IO/BufferedSubStream.cs index 0de8822c..08e60e15 100755 --- a/src/SharpCompress/IO/BufferedSubStream.cs +++ b/src/SharpCompress/IO/BufferedSubStream.cs @@ -73,6 +73,28 @@ internal partial class BufferedSubStream : Stream, IStreamStack return _cache![_cacheOffset++]; } + public override int Read(byte[] buffer, int offset, int count) + { + if (count > Length) + { + count = (int)Length; + } + + if (count > 0) + { + if (_cacheOffset == _cacheLength) + { + RefillCache(); + } + + count = Math.Min(count, _cacheLength - _cacheOffset); + Buffer.BlockCopy(_cache!, _cacheOffset, buffer, offset, count); + _cacheOffset += count; + } + + return count; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); diff --git a/src/SharpCompress/IO/ConfiguredAsyncDisposeScope.cs b/src/SharpCompress/IO/ConfiguredAsyncDisposeScope.cs new file mode 100644 index 00000000..9ee5e368 --- /dev/null +++ b/src/SharpCompress/IO/ConfiguredAsyncDisposeScope.cs @@ -0,0 +1,22 @@ +using System; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +internal readonly struct ConfiguredAsyncDisposeScope( + IDisposable? resource, + bool continueOnCapturedContext +) +{ + public ConfiguredValueTaskAwaitable DisposeAsync() + { + if (resource is IAsyncDisposable asyncDisposable) + { + return asyncDisposable.DisposeAsync().ConfigureAwait(continueOnCapturedContext); + } + + resource?.Dispose(); + return default(ValueTask).ConfigureAwait(continueOnCapturedContext); + } +} diff --git a/src/SharpCompress/IO/SeekableSharpCompressStream.cs b/src/SharpCompress/IO/SeekableSharpCompressStream.cs index 73d23aee..15e95fd8 100644 --- a/src/SharpCompress/IO/SeekableSharpCompressStream.cs +++ b/src/SharpCompress/IO/SeekableSharpCompressStream.cs @@ -67,7 +67,16 @@ internal sealed partial class SeekableSharpCompressStream : SharpCompressStream public override void StartRecording(int? minBufferSize = null) => _recordedPosition = _stream.Position; - public override void StopRecording() => _recordedPosition = null; + public override void StopRecording() + { + if (_recordedPosition.HasValue) + { + // Seek back to the recording anchor position, matching the behavior of the + // non-seekable SharpCompressStream.StopRecording() which rewinds _logicalPosition. + _stream.Seek(_recordedPosition.Value, SeekOrigin.Begin); + } + _recordedPosition = null; + } protected override void Dispose(bool disposing) { diff --git a/src/SharpCompress/Polyfills/StreamExtensions.cs b/src/SharpCompress/Polyfills/StreamExtensions.cs index c6e66b82..d4e80630 100644 --- a/src/SharpCompress/Polyfills/StreamExtensions.cs +++ b/src/SharpCompress/Polyfills/StreamExtensions.cs @@ -25,6 +25,20 @@ public static class StreamExtensions public void Skip() => stream.CopyTo(Stream.Null); + /// + /// Returns a scope that disposes this stream when awaited, asynchronously where the runtime type + /// supports it. Lets await using be written against a -typed local on + /// every target framework. + /// + internal AsyncDisposeScope DisposeAsyncScope() => new(stream); + + /// + /// Disposes this stream, asynchronously where the runtime type supports it. Use where the static + /// type is , which has no DisposeAsync on .NET Framework 4.8 / + /// .NET Standard 2.0. + /// + internal ValueTask DisposeAsyncCompat() => new AsyncDisposeScope(stream).DisposeAsync(); + public async ValueTask SkipAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/SharpCompress/Providers/CompressionProviderRegistry.cs b/src/SharpCompress/Providers/CompressionProviderRegistry.cs index ea0e217a..289487a0 100644 --- a/src/SharpCompress/Providers/CompressionProviderRegistry.cs +++ b/src/SharpCompress/Providers/CompressionProviderRegistry.cs @@ -26,7 +26,7 @@ namespace SharpCompress.Providers; /// }; /// /// -public sealed class CompressionProviderRegistry +public sealed partial class CompressionProviderRegistry { /// /// The default registry using SharpCompress internal implementations. @@ -61,103 +61,11 @@ public sealed class CompressionProviderRegistry /// The compression type. /// The destination stream. /// The compression level. - /// A compression stream. - /// If no provider is registered for the type. - /// If the provider does not support compression. - public Stream CreateCompressStream(CompressionType type, Stream destination, int level) - { - var provider = GetProvider(type); - if (provider is null) - { - throw new ArchiveOperationException( - $"No compression provider registered for type: {type}" - ); - } - return provider.CreateCompressStream(destination, level); - } - - /// - /// Creates a decompression stream for the specified type. - /// - /// The compression type. - /// The source stream. - /// A decompression stream. - /// If no provider is registered for the type. - /// If the provider does not support decompression. - public Stream CreateDecompressStream(CompressionType type, Stream source) - { - var provider = GetProvider(type); - if (provider is null) - { - throw new ArchiveOperationException( - $"No compression provider registered for type: {type}" - ); - } - return provider.CreateDecompressStream(source); - } - - /// - /// Creates a compression stream for the specified type with context. - /// - /// The compression type. - /// The destination stream. - /// The compression level. - /// Context information for the compression. - /// A compression stream. - /// If no provider is registered for the type. - /// If the provider does not support compression. - public Stream CreateCompressStream( - CompressionType type, - Stream destination, - int level, - CompressionContext context - ) - { - var provider = GetProvider(type); - if (provider is null) - { - throw new ArchiveOperationException( - $"No compression provider registered for type: {type}" - ); - } - return provider.CreateCompressStream(destination, level, context); - } - - /// - /// Creates a decompression stream for the specified type with context. - /// - /// The compression type. - /// The source stream. - /// Context information for the decompression. - /// A decompression stream. - /// If no provider is registered for the type. - /// If the provider does not support decompression. - public Stream CreateDecompressStream( - CompressionType type, - Stream source, - CompressionContext context - ) - { - var provider = GetProvider(type); - if (provider is null) - { - throw new ArchiveOperationException( - $"No compression provider registered for type: {type}" - ); - } - return provider.CreateDecompressStream(source, context); - } - - /// - /// Asynchronously creates a compression stream for the specified type. - /// - /// The compression type. - /// The destination stream. - /// The compression level. /// Cancellation token. - /// A task containing the compression stream. + /// A compression stream. /// If no provider is registered for the type. /// If the provider does not support compression. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public ValueTask CreateCompressStreamAsync( CompressionType type, Stream destination, @@ -176,14 +84,15 @@ public sealed class CompressionProviderRegistry } /// - /// Asynchronously creates a decompression stream for the specified type. + /// Creates a decompression stream for the specified type. /// /// The compression type. /// The source stream. /// Cancellation token. - /// A task containing the decompression stream. + /// A decompression stream. /// If no provider is registered for the type. /// If the provider does not support decompression. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public ValueTask CreateDecompressStreamAsync( CompressionType type, Stream source, @@ -201,16 +110,17 @@ public sealed class CompressionProviderRegistry } /// - /// Asynchronously creates a compression stream for the specified type with context. + /// Creates a compression stream for the specified type with context. /// /// The compression type. /// The destination stream. /// The compression level. /// Context information for the compression. /// Cancellation token. - /// A task containing the compression stream. + /// A compression stream. /// If no provider is registered for the type. /// If the provider does not support compression. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public ValueTask CreateCompressStreamAsync( CompressionType type, Stream destination, @@ -230,15 +140,16 @@ public sealed class CompressionProviderRegistry } /// - /// Asynchronously creates a decompression stream for the specified type with context. + /// Creates a decompression stream for the specified type with context. /// /// The compression type. /// The source stream. /// Context information for the decompression. /// Cancellation token. - /// A task containing the decompression stream. + /// A decompression stream. /// If no provider is registered for the type. /// If the provider does not support decompression. + [Zomp.SyncMethodGenerator.CreateSyncVersion] public ValueTask CreateDecompressStreamAsync( CompressionType type, Stream source, diff --git a/src/SharpCompress/Providers/Default/GZipCompressionProvider.cs b/src/SharpCompress/Providers/Default/GZipCompressionProvider.cs index 1365e0a7..be88c40c 100644 --- a/src/SharpCompress/Providers/Default/GZipCompressionProvider.cs +++ b/src/SharpCompress/Providers/Default/GZipCompressionProvider.cs @@ -11,7 +11,7 @@ namespace SharpCompress.Providers.Default; /// /// Provides GZip compression using SharpCompress's internal implementation. /// -public sealed class GZipCompressionProvider : CompressionProviderBase +public sealed partial class GZipCompressionProvider : CompressionProviderBase { public override CompressionType CompressionType => CompressionType.GZip; public override bool SupportsCompression => true; @@ -28,23 +28,22 @@ public sealed class GZipCompressionProvider : CompressionProviderBase return new GZipStream(source, CompressionMode.Decompress); } - public override Stream CreateDecompressStream(Stream source, CompressionContext context) - { - return new GZipStream( - source, - CompressionMode.Decompress, - CompressionLevel.Default, - context.ResolveArchiveEncoding() - ); - } - - public override ValueTask CreateDecompressStreamAsync( + [Zomp.SyncMethodGenerator.CreateSyncVersion] + public override async ValueTask CreateDecompressStreamAsync( Stream source, CompressionContext context, CancellationToken cancellationToken = default ) { cancellationToken.ThrowIfCancellationRequested(); - return new ValueTask(CreateDecompressStream(source, context)); + return await Task.FromResult( + new GZipStream( + source, + CompressionMode.Decompress, + CompressionLevel.Default, + context.ResolveArchiveEncoding() + ) + ) + .ConfigureAwait(false); } } diff --git a/src/SharpCompress/Readers/AbstractReader.Async.cs b/src/SharpCompress/Readers/AbstractReader.Async.cs index 8628c0c2..969c73e6 100644 --- a/src/SharpCompress/Readers/AbstractReader.Async.cs +++ b/src/SharpCompress/Readers/AbstractReader.Async.cs @@ -104,13 +104,8 @@ public abstract partial class AbstractReader } } //don't know the size so we have to try to decompress to skip -#if LEGACY_DOTNET - using var s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); - await s.SkipEntryAsync(cancellationToken).ConfigureAwait(false); -#else await using var s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); await s.SkipEntryAsync(cancellationToken).ConfigureAwait(false); -#endif } public async ValueTask WriteEntryToAsync( @@ -139,19 +134,11 @@ public abstract partial class AbstractReader private async ValueTask WriteAsync(Stream writeStream, CancellationToken cancellationToken) { -#if LEGACY_DOTNET - using Stream s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); + await using var s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); var sourceStream = WrapWithProgress(s, Entry); await sourceStream .CopyToAsync(writeStream, Options.BufferSize, cancellationToken) .ConfigureAwait(false); -#else - await using Stream s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); - var sourceStream = WrapWithProgress(s, Entry); - await sourceStream - .CopyToAsync(writeStream, Options.BufferSize, cancellationToken) - .ConfigureAwait(false); -#endif } public async ValueTask OpenEntryStreamAsync( diff --git a/src/SharpCompress/Readers/GZip/GZipReader.Async.cs b/src/SharpCompress/Readers/GZip/GZipReader.Async.cs index 720cb792..28b10b81 100644 --- a/src/SharpCompress/Readers/GZip/GZipReader.Async.cs +++ b/src/SharpCompress/Readers/GZip/GZipReader.Async.cs @@ -10,6 +10,7 @@ public partial class GZipReader /// /// Returns entries asynchronously for streams that only support async reads. /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] protected override IAsyncEnumerable GetEntriesAsync(Stream stream) => GZipEntry.GetEntriesAsync(stream, Options); } diff --git a/src/SharpCompress/Readers/GZip/GZipReader.cs b/src/SharpCompress/Readers/GZip/GZipReader.cs index eb09ab6c..aea23e73 100644 --- a/src/SharpCompress/Readers/GZip/GZipReader.cs +++ b/src/SharpCompress/Readers/GZip/GZipReader.cs @@ -11,9 +11,4 @@ public partial class GZipReader : AbstractReader : base(options, ArchiveType.GZip) => Volume = new GZipVolume(stream, options, 0); public override GZipVolume Volume { get; } - - protected override IEnumerable GetEntries(Stream stream) => - GZipEntry.GetEntries(stream, Options); - - // GetEntriesAsync moved to GZipReader.Async.cs } diff --git a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs index 7515b257..5b4e667b 100644 --- a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs +++ b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs @@ -108,15 +108,9 @@ public static class IAsyncReaderExtensions CancellationToken cancellationToken ) { -#if LEGACY_DOTNET - using var entryStream = await reader - .OpenEntryStreamAsync(cancellationToken) - .ConfigureAwait(false); -#else await using var entryStream = await reader .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); -#endif var checkedStream = IEntryExtensions.WrapWithChecksumValidation( reader.Entry, entryStream, diff --git a/src/SharpCompress/Readers/ReaderOptions.cs b/src/SharpCompress/Readers/ReaderOptions.cs index aced7169..3d9028b2 100644 --- a/src/SharpCompress/Readers/ReaderOptions.cs +++ b/src/SharpCompress/Readers/ReaderOptions.cs @@ -143,6 +143,13 @@ public sealed record ReaderOptions : IReaderOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel decode (e.g. 7-Zip's automatic + /// parallel LZMA2 solid-folder decode). Formats that do not implement parallel decoding + /// ignore this setting. Default is false (sequential decode). + /// + public bool EnableParallelism { get; set; } + /// /// Creates a new ReaderOptions instance with default values. /// diff --git a/src/SharpCompress/Readers/ReaderOptionsExtensions.cs b/src/SharpCompress/Readers/ReaderOptionsExtensions.cs index 2415b544..0e3628ab 100644 --- a/src/SharpCompress/Readers/ReaderOptionsExtensions.cs +++ b/src/SharpCompress/Readers/ReaderOptionsExtensions.cs @@ -86,6 +86,14 @@ public static class ReaderOptionsExtensions int? rewindableBufferSize ) => options with { RewindableBufferSize = rewindableBufferSize }; + /// + /// Creates a copy with the specified EnableParallelism value. + /// + public static ReaderOptions WithEnableParallelism( + this ReaderOptions options, + bool enableParallelism + ) => options with { EnableParallelism = enableParallelism }; + /// /// Creates a copy with the specified compression provider registry. /// diff --git a/src/SharpCompress/SharpCompress.csproj b/src/SharpCompress/SharpCompress.csproj index 991eccb9..c57cb722 100644 --- a/src/SharpCompress/SharpCompress.csproj +++ b/src/SharpCompress/SharpCompress.csproj @@ -31,6 +31,9 @@ $(DefineConstants);LEGACY_DOTNET + + $(DefineConstants);NO_STREAM_DISPOSEASYNC + true true diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index c03c3643..c5824d51 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -23,15 +23,6 @@ internal static partial class Utility ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - public static bool UseSyncOverAsyncDispose() - { - var useSyncOverAsync = false; -#if LEGACY_DOTNET - useSyncOverAsync = true; -#endif - return useSyncOverAsync; - } - private static readonly HashSet invalidChars = new(Path.GetInvalidFileNameChars()); public static ReadOnlyCollection ToReadOnly(this IList items) => new(items); diff --git a/src/SharpCompress/Writers/GZip/GZipWriter.Async.cs b/src/SharpCompress/Writers/GZip/GZipWriter.Async.cs index a1aa613a..de72c0d4 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriter.Async.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriter.Async.cs @@ -3,11 +3,13 @@ using System.IO; using System.Threading; using System.Threading.Tasks; using SharpCompress.Compressors.Deflate; +using SharpCompress.IO; namespace SharpCompress.Writers.GZip; public partial class GZipWriter { + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async ValueTask WriteAsync( string filename, Stream source, @@ -19,18 +21,28 @@ public partial class GZipWriter { throw new ArgumentException("Can only write a single stream to a GZip file."); } - var stream = (GZipStream)OutputStream.NotNull(); - stream.FileName = filename; - stream.LastModified = modificationTime; + + // Custom providers need not expose SharpCompress's internal GZip stream. + if (OutputStream is GZipStream gzipStream) + { + gzipStream.FileName = filename; + gzipStream.LastModified = modificationTime; + } + var progressStream = WrapWithProgress(source, filename); #if LEGACY_DOTNET - await progressStream.CopyToAsync(stream).ConfigureAwait(false); + await progressStream + .CopyToAsync(OutputStream.NotNull(), WriterOptions.BufferSize) + .ConfigureAwait(false); #else - await progressStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); + await progressStream + .CopyToAsync(OutputStream.NotNull(), WriterOptions.BufferSize, cancellationToken) + .ConfigureAwait(false); #endif _wroteToStream = true; } + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override ValueTask WriteDirectoryAsync( string directoryName, DateTime? modificationTime, diff --git a/src/SharpCompress/Writers/GZip/GZipWriter.cs b/src/SharpCompress/Writers/GZip/GZipWriter.cs index 3c62ff2d..604e7b02 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriter.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriter.cs @@ -70,26 +70,4 @@ public sealed partial class GZipWriter : AbstractWriter } } #pragma warning restore CA2215 - - public override void Write(string filename, Stream source, DateTime? modificationTime) - { - if (_wroteToStream) - { - throw new ArgumentException("Can only write a single stream to a GZip file."); - } - - // Set metadata on the stream if it's the internal GZipStream - if (OutputStream is GZipStream gzipStream) - { - gzipStream.FileName = filename; - gzipStream.LastModified = modificationTime; - } - - var progressStream = WrapWithProgress(source, filename); - progressStream.CopyTo(OutputStream.NotNull(), WriterOptions.BufferSize); - _wroteToStream = true; - } - - public override void WriteDirectory(string directoryName, DateTime? modificationTime) => - throw new NotSupportedException("GZip archives do not support directory entries."); } diff --git a/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs b/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs index ca1b6a9d..e6c4b3bb 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs @@ -81,6 +81,12 @@ public sealed record GZipWriterOptions : IWriterOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel encode. No writer currently implements + /// parallel encoding; reserved for future use. Default is false. + /// + public bool EnableParallelism { get; set; } + /// /// Creates a new GZipWriterOptions instance with default values. /// diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs index 6ed26a42..2a286434 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs @@ -69,6 +69,12 @@ public sealed record SevenZipWriterOptions : IWriterOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel encode. No writer currently implements + /// parallel encoding; reserved for future use. Default is false. + /// + public bool EnableParallelism { get; set; } + /// /// Whether to compress the archive header itself using LZMA. /// Default is true, matching standard 7-Zip behavior. diff --git a/src/SharpCompress/Writers/Tar/TarWriterOptions.cs b/src/SharpCompress/Writers/Tar/TarWriterOptions.cs index d7aaf22b..369df364 100755 --- a/src/SharpCompress/Writers/Tar/TarWriterOptions.cs +++ b/src/SharpCompress/Writers/Tar/TarWriterOptions.cs @@ -56,6 +56,12 @@ public sealed record TarWriterOptions : IWriterOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel encode. No writer currently implements + /// parallel encoding; reserved for future use. Default is false. + /// + public bool EnableParallelism { get; set; } + /// /// Indicates if archive should be finalized (by 2 empty blocks) on close. /// diff --git a/src/SharpCompress/Writers/WriterOptions.cs b/src/SharpCompress/Writers/WriterOptions.cs index d70ea6aa..0b70df88 100644 --- a/src/SharpCompress/Writers/WriterOptions.cs +++ b/src/SharpCompress/Writers/WriterOptions.cs @@ -69,6 +69,12 @@ public sealed record WriterOptions : IWriterOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel encode. No writer currently implements + /// parallel encoding; reserved for future use. Default is false. + /// + public bool EnableParallelism { get; set; } + /// /// Creates a new WriterOptions instance with the specified compression type. /// Compression level is automatically set based on the compression type. diff --git a/src/SharpCompress/Writers/WriterOptionsExtensions.cs b/src/SharpCompress/Writers/WriterOptionsExtensions.cs index 2aca567a..c7689c00 100644 --- a/src/SharpCompress/Writers/WriterOptionsExtensions.cs +++ b/src/SharpCompress/Writers/WriterOptionsExtensions.cs @@ -73,6 +73,14 @@ public static class WriterOptionsExtensions int compressionLevel ) => options with { CompressionLevel = compressionLevel }; + /// + /// Creates a copy with the specified EnableParallelism value. + /// + public static WriterOptions WithEnableParallelism( + this WriterOptions options, + bool enableParallelism + ) => options with { EnableParallelism = enableParallelism }; + /// /// Creates a copy with the specified archive encoding. /// diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs index b934c6e1..ec0d9ca4 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.Async.cs @@ -47,8 +47,9 @@ public partial class ZipWriter } /// - /// Asynchronously writes an entry to the ZIP archive. + /// Writes an entry to the ZIP archive. /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async ValueTask WriteAsync( string filename, Stream source, @@ -56,7 +57,9 @@ public partial class ZipWriter CancellationToken cancellationToken = default ) { +#if !SYNC_ONLY cancellationToken.ThrowIfCancellationRequested(); +#endif await WriteAsync( filename, source, @@ -67,8 +70,9 @@ public partial class ZipWriter } /// - /// Asynchronously writes an entry to the ZIP archive with specified options. + /// Writes an entry to the ZIP archive with specified options. /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public async ValueTask WriteAsync( string entryPath, Stream source, @@ -76,7 +80,9 @@ public partial class ZipWriter CancellationToken cancellationToken = default ) { +#if !SYNC_ONLY cancellationToken.ThrowIfCancellationRequested(); +#endif await using var output = await WriteToStreamAsync( entryPath, zipWriterEntryOptions, @@ -84,7 +90,9 @@ public partial class ZipWriter ) .ConfigureAwait(false); var progressStream = WrapWithProgress(source, entryPath); - await progressStream.CopyToAsync(output, 81920, cancellationToken).ConfigureAwait(false); + await progressStream + .CopyToAsync(output, WriterOptions.BufferSize, cancellationToken) + .ConfigureAwait(false); } private async ValueTask WriteToStreamAsync( @@ -168,16 +176,18 @@ public partial class ZipWriter } /// - /// Asynchronously writes a directory entry to the ZIP archive. - /// Uses synchronous implementation for directory entries as they are lightweight. + /// Writes a directory entry to the ZIP archive. /// + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async ValueTask WriteDirectoryAsync( string directoryName, DateTime? modificationTime, CancellationToken cancellationToken = default ) { +#if !SYNC_ONLY cancellationToken.ThrowIfCancellationRequested(); +#endif var normalizedName = NormalizeDirectoryName(directoryName); if (string.IsNullOrEmpty(normalizedName)) @@ -190,6 +200,7 @@ public partial class ZipWriter .ConfigureAwait(false); } + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask WriteDirectoryEntryAsync( string directoryPath, ZipWriterEntryOptions options, diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index 8775253c..f9640c32 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -77,20 +77,6 @@ public partial class ZipWriter : AbstractWriter _ => throw new InvalidFormatException("Invalid compression method: " + compressionType), }; - public override void Write(string filename, Stream source, DateTime? modificationTime) => - Write( - filename, - source, - new ZipWriterEntryOptions() { ModificationDateTime = modificationTime } - ); - - public void Write(string entryPath, Stream source, ZipWriterEntryOptions zipWriterEntryOptions) - { - using var output = WriteToStream(entryPath, zipWriterEntryOptions); - var progressStream = WrapWithProgress(source, entryPath); - progressStream.CopyTo(output, WriterOptions.BufferSize); - } - public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options) { options.ValidateWithFallback(compressionType, compressionLevel); @@ -162,59 +148,6 @@ public partial class ZipWriter : AbstractWriter return directoryName; } - public override void WriteDirectory(string directoryName, DateTime? modificationTime) - { - var normalizedName = NormalizeDirectoryName(directoryName); - if (string.IsNullOrEmpty(normalizedName)) - { - return; // Skip empty or root directory - } - - var options = new ZipWriterEntryOptions { ModificationDateTime = modificationTime }; - WriteDirectoryEntry(normalizedName, options); - } - - // WriteDirectoryAsync moved to ZipWriter.Async.cs - - private void WriteDirectoryEntry(string directoryPath, ZipWriterEntryOptions options) - { - var compression = ZipCompressionMethod.None; - - options.ModificationDateTime ??= DateTime.Now; - options.EntryComment ??= string.Empty; - - var entry = new ZipCentralDirectoryEntry( - compression, - directoryPath, - (ulong)streamPosition, - WriterOptions.ArchiveEncoding - ) - { - Comment = options.EntryComment, - ModificationTime = options.ModificationDateTime, - Crc = 0, - Compressed = 0, - Decompressed = 0, - }; - - // Use the archive default setting for zip64 and allow overrides - var useZip64 = isZip64; - if (options.EnableZip64.HasValue) - { - useZip64 = options.EnableZip64.Value; - } - - var headersize = (uint)WriteHeader( - directoryPath, - options, - entry, - useZip64, - usesDataDescriptor: false - ); - streamPosition += headersize; - entries.Add(entry); - } - private int WriteHeader( string filename, ZipWriterEntryOptions zipWriterEntryOptions, diff --git a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs index 73c92ee6..fc7d613b 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs @@ -73,6 +73,12 @@ public sealed record ZipWriterOptions : IWriterOptions public CompressionProviderRegistry Providers { get; set; } = CompressionProviderRegistry.Default; + /// + /// When true, opts in to a format's optional parallel encode. No writer currently implements + /// parallel encoding; reserved for future use. Default is false. + /// + public bool EnableParallelism { get; set; } + /// /// Optional comment for the archive. /// diff --git a/src/SharpCompress/Writers/Zip/ZipWritingStream.cs b/src/SharpCompress/Writers/Zip/ZipWritingStream.cs index ad7c7473..e84b5155 100644 --- a/src/SharpCompress/Writers/Zip/ZipWritingStream.cs +++ b/src/SharpCompress/Writers/Zip/ZipWritingStream.cs @@ -15,7 +15,7 @@ namespace SharpCompress.Writers.Zip; public partial class ZipWriter { - internal class ZipWritingStream : Stream + internal partial class ZipWritingStream : Stream { private readonly CRC32 crc = new(); private readonly ZipCentralDirectoryEntry entry; @@ -98,114 +98,7 @@ public partial class ZipWriter set => throw new NotSupportedException(); } - private Stream GetWriteStream(Stream writeStream) - { - counting = new CountingStream(SharpCompressStream.CreateNonDisposing(writeStream)); - Stream output = counting; - - var providers = writer.WriterOptions.Providers; - - switch (zipCompressionMethod) - { - case ZipCompressionMethod.None: - { - return output; - } - case ZipCompressionMethod.Deflate: - { - return providers.CreateCompressStream( - CompressionType.Deflate, - counting, - compressionLevel - ); - } - case ZipCompressionMethod.BZip2: - { - return providers.CreateCompressStream( - CompressionType.BZip2, - counting, - compressionLevel - ); - } - case ZipCompressionMethod.LZMA: - { - var compressingProvider = providers.GetCompressingProvider( - CompressionType.LZMA - ); - if (compressingProvider is null) - { - throw new ArchiveOperationException("LZMA compression provider not found."); - } - - var context = new CompressionContext { CanSeek = originalStream.CanSeek }; - compressionProviderHooks = compressingProvider; - compressionContext = context; - - var preData = compressingProvider.GetPreCompressionData(context); - if (preData is not null) - { - counting.Write(preData, 0, preData.Length); - } - - var lzmaStream = compressingProvider.CreateCompressStream( - counting, - compressionLevel, - context - ); - - var props = compressingProvider.GetCompressionProperties(lzmaStream, context); - if (props is not null) - { - counting.Write(props, 0, props.Length); - } - - return lzmaStream; - } - case ZipCompressionMethod.PPMd: - { - var compressingProvider = providers.GetCompressingProvider( - CompressionType.PPMd - ); - if (compressingProvider is null) - { - throw new ArchiveOperationException("PPMd compression provider not found."); - } - - var context = new CompressionContext - { - CanSeek = originalStream.CanSeek, - FormatOptions = writer.PpmdProperties, - }; - compressionProviderHooks = compressingProvider; - compressionContext = context; - - var preData = compressingProvider.GetPreCompressionData(context); - if (preData is not null) - { - counting.Write(preData, 0, preData.Length); - } - - return compressingProvider.CreateCompressStream( - counting, - compressionLevel, - context - ); - } - case ZipCompressionMethod.ZStandard: - { - return providers.CreateCompressStream( - CompressionType.ZStandard, - counting, - compressionLevel - ); - } - default: - { - throw new NotSupportedException("CompressionMethod: " + zipCompressionMethod); - } - } - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask GetWriteStreamAsync( Stream writeStream, CancellationToken cancellationToken @@ -459,17 +352,7 @@ public partial class ZipWriter public override void SetLength(long value) => throw new NotSupportedException(); - public override void Write(byte[] buffer, int offset, int count) - { - CheckWriteLimits(count); - - decompressed += (uint)count; - crc.SlurpBlock(buffer, offset, count); - writeStream.Write(buffer, offset, count); - - CheckPostWriteLimits(); - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] public override async Task WriteAsync( byte[] buffer, int offset, @@ -477,7 +360,9 @@ public partial class ZipWriter CancellationToken cancellationToken ) { +#if !SYNC_ONLY cancellationToken.ThrowIfCancellationRequested(); +#endif CheckWriteLimits(count); decompressed += (uint)count; @@ -556,30 +441,7 @@ public partial class ZipWriter } } - private void WritePostCompressionData() - { - if ( - compressionProviderHooks is null - || compressionContext is null - || counting is null - || zipCompressionMethod == ZipCompressionMethod.None - ) - { - return; - } - - var postData = compressionProviderHooks.GetPostCompressionData( - writeStream, - compressionContext - ); - if (postData is null || postData.Length == 0) - { - return; - } - - counting.Write(postData, 0, postData.Length); - } - + [Zomp.SyncMethodGenerator.CreateSyncVersion] private async ValueTask WritePostCompressionDataAsync(CancellationToken cancellationToken) { if ( diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index 000d3159..698a290d 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -22,13 +22,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -55,16 +55,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net48": { @@ -74,8 +74,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.Buffers": { "type": "Transitive", @@ -84,8 +84,8 @@ }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==", + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==", "dependencies": { "System.Buffers": "4.6.1", "System.Memory": "4.6.3" @@ -141,13 +141,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -183,16 +183,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETCore.Platforms": { @@ -207,8 +207,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.Buffers": { "type": "Transitive", @@ -217,8 +217,8 @@ }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==", + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==", "dependencies": { "System.Buffers": "4.6.1", "System.Memory": "4.6.3" @@ -271,13 +271,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -303,16 +303,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -322,13 +322,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", @@ -354,13 +354,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -377,16 +377,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -396,13 +396,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" } }, "net6.0": { @@ -417,13 +417,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -440,16 +440,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -459,13 +459,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" } }, "net8.0": { @@ -486,13 +486,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -509,16 +509,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -528,13 +528,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" } } } diff --git a/tests/SharpCompress.AotSmoke/packages.lock.json b/tests/SharpCompress.AotSmoke/packages.lock.json index 4753b4a8..8f4a2893 100644 --- a/tests/SharpCompress.AotSmoke/packages.lock.json +++ b/tests/SharpCompress.AotSmoke/packages.lock.json @@ -25,13 +25,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -48,16 +48,16 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { @@ -67,13 +67,13 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" }, "sharpcompress": { "type": "Project" diff --git a/tests/SharpCompress.Performance/packages.lock.json b/tests/SharpCompress.Performance/packages.lock.json index f87fe2fc..33b82c41 100644 --- a/tests/SharpCompress.Performance/packages.lock.json +++ b/tests/SharpCompress.Performance/packages.lock.json @@ -41,13 +41,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -64,9 +64,9 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "BenchmarkDotNet.Annotations": { "type": "Transitive", @@ -111,10 +111,10 @@ }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.CodeAnalysis.Analyzers": { @@ -219,8 +219,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "Perfolizer": { "type": "Transitive", @@ -242,8 +242,8 @@ }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" }, "System.Management": { "type": "Transitive", diff --git a/tests/SharpCompress.Test/ArchiveFactoryTests.cs b/tests/SharpCompress.Test/ArchiveFactoryTests.cs index 199dff47..85fbca99 100644 --- a/tests/SharpCompress.Test/ArchiveFactoryTests.cs +++ b/tests/SharpCompress.Test/ArchiveFactoryTests.cs @@ -4,6 +4,7 @@ using System.Text; using System.Threading.Tasks; using SharpCompress.Archives; using SharpCompress.Common; +using SharpCompress.Detection; using SharpCompress.Factories; using SharpCompress.Readers; using SharpCompress.Test.Mocks; @@ -247,38 +248,36 @@ public class ArchiveFactoryTests : TestBase [InlineData("7Zip.nonsolid.7z", ArchiveType.SevenZip, true)] [InlineData("Ace.store.ace", ArchiveType.Ace, false)] [InlineData("Arc.uncompressed.arc", ArchiveType.Arc, false)] - public void GetArchiveInformation_ReturnsExpectedInfo( + public void DetectArchive_ReturnsExpectedInfo( string archiveName, ArchiveType expectedType, bool expectedRandomAccess ) { - var info = ArchiveFactory.GetArchiveInformation( - Path.Combine(TEST_ARCHIVES_PATH, archiveName) - ); + var info = ArchiveFactory.DetectArchive(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedRandomAccess, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedRandomAccess, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Theory] [InlineData("7Zip.LZMA2.exe", ArchiveType.SevenZip, true)] [InlineData("Rar.jpeg.jpg", ArchiveType.Rar, true)] - public void GetArchiveInformation_WithReaderOptions_ReturnsExpectedInfo( + public void DetectArchive_WithReaderOptions_ReturnsExpectedInfo( string archiveName, ArchiveType expectedType, bool expectedRandomAccess ) { - var info = ArchiveFactory.GetArchiveInformation( + var info = ArchiveFactory.DetectArchive( GetTestArchivePath(archiveName), ReaderOptions.ForFilePath.WithLookForHeader(true) ); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedRandomAccess, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedRandomAccess, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Theory] @@ -288,38 +287,38 @@ public class ArchiveFactoryTests : TestBase [InlineData("7Zip.nonsolid.7z", ArchiveType.SevenZip, true)] [InlineData("Ace.store.ace", ArchiveType.Ace, false)] [InlineData("Arc.uncompressed.arc", ArchiveType.Arc, false)] - public async ValueTask GetArchiveInformationAsync_ReturnsExpectedInfo( + public async ValueTask DetectArchiveAsync_ReturnsExpectedInfo( string archiveName, ArchiveType expectedType, bool expectedRandomAccess ) { - var info = await ArchiveFactory.GetArchiveInformationAsync( + var info = await ArchiveFactory.DetectArchiveAsync( Path.Combine(TEST_ARCHIVES_PATH, archiveName) ); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedRandomAccess, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedRandomAccess, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Theory] [InlineData("7Zip.LZMA2.exe", ArchiveType.SevenZip, true)] [InlineData("Rar.jpeg.jpg", ArchiveType.Rar, true)] - public async ValueTask GetArchiveInformationAsync_WithReaderOptions_ReturnsExpectedInfo( + public async ValueTask DetectArchiveAsync_WithReaderOptions_ReturnsExpectedInfo( string archiveName, ArchiveType expectedType, bool expectedRandomAccess ) { - var info = await ArchiveFactory.GetArchiveInformationAsync( + var info = await ArchiveFactory.DetectArchiveAsync( GetTestArchivePath(archiveName), ReaderOptions.ForFilePath.WithLookForHeader(true) ); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedRandomAccess, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedRandomAccess, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Theory] @@ -466,17 +465,17 @@ public class ArchiveFactoryTests : TestBase [InlineData("SharpCompress.AES.zip", ArchiveType.Zip, true)] [InlineData("SharpCompress.Encrypted.zip", ArchiveType.Zip, true)] [InlineData("SharpCompress.Encrypted2.zip", ArchiveType.Zip, true)] - public void GetArchiveInformation_DetectsSingleFileTestArchives( + public void DetectArchive_DetectsSingleFileTestArchives( string archiveName, ArchiveType expectedType, bool expectedSeekable ) { - var info = ArchiveFactory.GetArchiveInformation(GetTestArchivePath(archiveName)); + var info = ArchiveFactory.DetectArchive(GetTestArchivePath(archiveName)); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedSeekable, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedSeekable, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Theory] @@ -623,35 +622,35 @@ public class ArchiveFactoryTests : TestBase [InlineData("SharpCompress.AES.zip", ArchiveType.Zip, true)] [InlineData("SharpCompress.Encrypted.zip", ArchiveType.Zip, true)] [InlineData("SharpCompress.Encrypted2.zip", ArchiveType.Zip, true)] - public async ValueTask GetArchiveInformationAsync_DetectsSingleFileTestArchives( + public async ValueTask DetectArchiveAsync_DetectsSingleFileTestArchives( string archiveName, ArchiveType expectedType, bool expectedSeekable ) { - var info = await ArchiveFactory.GetArchiveInformationAsync(GetTestArchivePath(archiveName)); + var info = await ArchiveFactory.DetectArchiveAsync(GetTestArchivePath(archiveName)); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); - Assert.Equal(expectedSeekable, info.SupportsRandomAccess); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(expectedSeekable, info.SupportedApis.HasFlag(ArchiveAccessMode.Archive)); } [Fact] - public void GetArchiveInformation_ReturnsNull_ForNonArchive() + public void DetectArchive_ReturnsNull_ForNonArchive() { using var stream = new MemoryStream(Encoding.ASCII.GetBytes("not an archive")); - var info = ArchiveFactory.GetArchiveInformation(stream); + var info = ArchiveFactory.DetectArchive(stream); Assert.Null(info); } [Fact] - public async ValueTask GetArchiveInformationAsync_ReturnsNull_ForNonArchive() + public async ValueTask DetectArchiveAsync_ReturnsNull_ForNonArchive() { using var stream = new MemoryStream(Encoding.ASCII.GetBytes("not an archive")); - var info = await ArchiveFactory.GetArchiveInformationAsync(stream); + var info = await ArchiveFactory.DetectArchiveAsync(stream); Assert.Null(info); } @@ -697,13 +696,16 @@ public class ArchiveFactoryTests : TestBase [InlineData("Tar.tar.xz")] [InlineData("Tar.tar.zst")] [InlineData("Tar.tar.Z")] - public void GetArchiveInformation_ReturnsNull_ForCompressedTar(string archiveName) + public void DetectArchive_RecognizesCompressedTar(string archiveName) { using var stream = File.OpenRead(GetTestArchivePath(archiveName)); - var info = ArchiveFactory.GetArchiveInformation(stream); + var info = ArchiveFactory.DetectArchive(stream); - Assert.Null(info); + Assert.NotNull(info); + Assert.Equal(ArchiveType.Tar, info.ContainerType); + Assert.NotNull(info.OuterCompressionType); + Assert.Equal(ArchiveAccessMode.Reader, info.SupportedApis); } [Theory] @@ -713,21 +715,131 @@ public class ArchiveFactoryTests : TestBase [InlineData("Tar.tar.xz")] [InlineData("Tar.tar.zst")] [InlineData("Tar.tar.Z")] - public async ValueTask GetArchiveInformationAsync_ReturnsNull_ForCompressedTar( - string archiveName + public async ValueTask DetectArchiveAsync_RecognizesCompressedTar(string archiveName) + { + using var stream = File.OpenRead(GetTestArchivePath(archiveName)); + + var info = await ArchiveFactory.DetectArchiveAsync(stream); + + Assert.NotNull(info); + Assert.Equal(ArchiveType.Tar, info.ContainerType); + Assert.NotNull(info.OuterCompressionType); + Assert.Equal(ArchiveAccessMode.Reader, info.SupportedApis); + } + + [Theory] + [InlineData("Zip.deflate.zip", ArchiveType.Zip)] + [InlineData("Tar.noEmptyDirs.tar", ArchiveType.Tar)] + public void DetectArchive_Stream_PreservesPosition(string archiveName, ArchiveType expectedType) + { + using var stream = CreatePrefixedArchiveStream(archiveName, 13); + var startPosition = stream.Position; + + var info = ArchiveFactory.DetectArchive(stream); + + Assert.NotNull(info); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(startPosition, stream.Position); + } + + [Theory] + [InlineData("Zip.deflate.zip", ArchiveType.Zip)] + [InlineData("Tar.noEmptyDirs.tar", ArchiveType.Tar)] + public async ValueTask DetectArchiveAsync_Stream_PreservesPosition( + string archiveName, + ArchiveType expectedType + ) + { + using var stream = CreatePrefixedArchiveStream(archiveName, 13); + var startPosition = stream.Position; + + var info = await ArchiveFactory.DetectArchiveAsync(stream); + + Assert.NotNull(info); + Assert.Equal(expectedType, info.ContainerType); + Assert.Equal(startPosition, stream.Position); + } + + [Theory] + [InlineData("Zip.none.datadescriptors.zip", true, 2)] + [InlineData("Zip.deflate.dd.zip", true, 3)] + [InlineData("Zip.deflate.zip", false, 0)] + public void InspectArchive_ZipReportsEntriesWithDeferredSizes( + string archiveName, + bool expectedHasEntriesWithDeferredSizes, + long expectedUnknownSizeEntryCount ) { using var stream = File.OpenRead(GetTestArchivePath(archiveName)); - var info = await ArchiveFactory.GetArchiveInformationAsync(stream); + var info = ArchiveFactory.InspectArchive(stream); - Assert.Null(info); + Assert.NotNull(info); + Assert.Equal(ArchiveType.Zip, info.Detection.ContainerType); + Assert.NotNull(info.Zip); + Assert.Equal(expectedHasEntriesWithDeferredSizes, info.Zip.HasEntriesWithDeferredSizes); + Assert.Equal(expectedUnknownSizeEntryCount, info.EntriesWithUnknownSizeCount); + } + + [Theory] + [InlineData("Zip.none.datadescriptors.zip", true)] + [InlineData("Zip.deflate.zip", false)] + public async ValueTask InspectArchiveAsync_ZipReportsEntriesWithDeferredSizes( + string archiveName, + bool expectedHasEntriesWithDeferredSizes + ) + { + var info = await ArchiveFactory.InspectArchiveAsync(GetTestArchivePath(archiveName)); + + Assert.NotNull(info); + Assert.NotNull(info.Zip); + Assert.Equal(expectedHasEntriesWithDeferredSizes, info.Zip.HasEntriesWithDeferredSizes); + } + + [Theory] + [InlineData("7Zip.solid.7z", true, 1)] + [InlineData("7Zip.nonsolid.7z", false, 0)] + [InlineData("Rar.rar", false, 0)] + [InlineData("Rar.solid.rar", true, 1)] + public void InspectArchive_SolidStreamCount( + string archiveName, + bool expectedSolid, + long expectedSolidStreamCount + ) + { + using var stream = File.OpenRead(GetTestArchivePath(archiveName)); + + var info = ArchiveFactory.InspectArchive(stream); + + Assert.NotNull(info); + Assert.Equal(expectedSolid, info.IsSolid); + Assert.Equal(expectedSolidStreamCount, info.SolidStreamCount); + } + + [Theory] + [InlineData("Zip.none.datadescriptors.zip", ArchiveType.Zip)] + [InlineData("7Zip.solid.7z", ArchiveType.SevenZip)] + public async ValueTask InspectArchiveAsync_ReturnsMetadata( + string archiveName, + ArchiveType expectedType + ) + { + using var stream = File.OpenRead(GetTestArchivePath(archiveName)); + + var info = await ArchiveFactory.InspectArchiveAsync(stream); + + Assert.NotNull(info); + Assert.Equal(expectedType, info.Detection.ContainerType); + Assert.Equal(ArchiveInformationStatus.Complete, info.Status); + Assert.NotNull(info.EntryCount); } [Theory] [InlineData("Zip.deflate.zip", ArchiveType.Zip)] [InlineData("Tar.noEmptyDirs.tar", ArchiveType.Tar)] - public void GetArchiveInformation_Stream_PreservesPosition( + [InlineData("Rar.rar", ArchiveType.Rar)] + [InlineData("7Zip.nonsolid.7z", ArchiveType.SevenZip)] + public void InspectArchive_Stream_PreservesPosition( string archiveName, ArchiveType expectedType ) @@ -735,17 +847,19 @@ public class ArchiveFactoryTests : TestBase using var stream = CreatePrefixedArchiveStream(archiveName, 13); var startPosition = stream.Position; - var info = ArchiveFactory.GetArchiveInformation(stream); + var info = ArchiveFactory.InspectArchive(stream); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); + Assert.Equal(expectedType, info.Detection.ContainerType); Assert.Equal(startPosition, stream.Position); } [Theory] [InlineData("Zip.deflate.zip", ArchiveType.Zip)] [InlineData("Tar.noEmptyDirs.tar", ArchiveType.Tar)] - public async ValueTask GetArchiveInformationAsync_Stream_PreservesPosition( + [InlineData("Rar.rar", ArchiveType.Rar)] + [InlineData("7Zip.nonsolid.7z", ArchiveType.SevenZip)] + public async ValueTask InspectArchiveAsync_Stream_PreservesPosition( string archiveName, ArchiveType expectedType ) @@ -753,13 +867,154 @@ public class ArchiveFactoryTests : TestBase using var stream = CreatePrefixedArchiveStream(archiveName, 13); var startPosition = stream.Position; - var info = await ArchiveFactory.GetArchiveInformationAsync(stream); + var info = await ArchiveFactory.InspectArchiveAsync(stream); Assert.NotNull(info); - Assert.Equal(expectedType, info.Type); + Assert.Equal(expectedType, info.Detection.ContainerType); Assert.Equal(startPosition, stream.Position); } + [Fact] + public void InspectArchive_CompressedTar_ReportsContainerAndWrapper() + { + var info = ArchiveFactory.InspectArchive(GetTestArchivePath("Tar.tar.xz")); + + Assert.NotNull(info); + Assert.Equal(ArchiveType.Tar, info.Detection.ContainerType); + Assert.Equal(CompressionType.Xz, info.Detection.OuterCompressionType); + Assert.Equal(ArchiveInformationStatus.Partial, info.Status); + Assert.Null(info.CompressedPayloadSize); + Assert.NotNull(info.EntryCount); + } + + [Theory] + [InlineData("Ace.method1-solid.ace", true)] + [InlineData("Ace.method1.ace", false)] + public void InspectArchive_Ace_ReportsSolidMetadata(string archiveName, bool expectedSolid) + { + var info = ArchiveFactory.InspectArchive(GetTestArchivePath(archiveName)); + + Assert.NotNull(info); + Assert.Equal(expectedSolid, info.IsSolid); + Assert.Equal(expectedSolid ? 1 : 0, info.SolidStreamCount); + } + + [Fact] + public void InspectArchive_Arc_ReportsUnavailableUncompressedSizes() + { + var info = ArchiveFactory.InspectArchive(GetTestArchivePath("Arc.uncompressed.arc")); + + Assert.NotNull(info); + Assert.Equal(ArchiveInformationStatus.Partial, info.Status); + Assert.True(info.Limitations.HasFlag(ArchiveInformationLimitations.UnavailableMetadata)); + Assert.Null(info.UncompressedPayloadSize); + Assert.True(info.EntriesWithUnknownSizeCount > 0); + } + + [Theory] + [InlineData("Arj.store.arj", ArchiveType.Arj, ArchiveInformationStatus.Complete)] + [InlineData("large_test.txt.Z", ArchiveType.Lzw, ArchiveInformationStatus.Partial)] + public void InspectArchive_ReaderOnlyFormats_ReturnsMetadata( + string archiveName, + ArchiveType expectedType, + ArchiveInformationStatus expectedStatus + ) + { + var info = ArchiveFactory.InspectArchive(GetTestArchivePath(archiveName)); + + Assert.NotNull(info); + Assert.Equal(expectedType, info.Detection.ContainerType); + Assert.Equal(expectedStatus, info.Status); + Assert.NotNull(info.EntryCount); + } + + [Fact] + public void InspectArchive_EncryptedHeaders_ReturnsPartialInformation() + { + var info = ArchiveFactory.InspectArchive( + GetTestArchivePath("Rar.encrypted_filesAndHeader.rar") + ); + + Assert.NotNull(info); + Assert.Equal(ArchiveInformationStatus.Partial, info.Status); + Assert.True(info.Limitations.HasFlag(ArchiveInformationLimitations.EncryptedHeaders)); + Assert.Equal(ArchiveEncryptionScope.Headers, info.Encryption); + Assert.True(info.IsEncrypted); + } + + [Fact] + public void InspectArchive_EncryptedHeadersWithPassword_ReportsBothEncryptionScopes() + { + var info = ArchiveFactory.InspectArchive( + GetTestArchivePath("Rar.encrypted_filesAndHeader.rar"), + ReaderOptions.ForFilePath.WithPassword("test") + ); + + Assert.NotNull(info); + Assert.Equal(ArchiveInformationStatus.Complete, info.Status); + Assert.Equal( + ArchiveEncryptionScope.Headers | ArchiveEncryptionScope.EntryData, + info.Encryption + ); + Assert.True(info.IsEncrypted); + } + + [Fact] + public void InspectArchive_IncompleteRarStream_ReturnsPartialInformation() + { + using var stream = File.OpenRead(GetTestArchivePath("Rar.multi.solid.part01.rar")); + + var info = ArchiveFactory.InspectArchive(stream); + + Assert.NotNull(info); + Assert.Equal(ArchiveInformationStatus.Partial, info.Status); + Assert.True(info.Limitations.HasFlag(ArchiveInformationLimitations.MissingVolumes)); + Assert.Null(info.CompressedPayloadSize); + Assert.Null(info.UncompressedPayloadSize); + } + + [Fact] + public async ValueTask InspectArchiveAsync_CompressedTar_ReturnsMetadata() + { + var info = await ArchiveFactory.InspectArchiveAsync(GetTestArchivePath("Tar.tar.xz")); + + Assert.NotNull(info); + Assert.Equal(ArchiveType.Tar, info.Detection.ContainerType); + Assert.Equal(CompressionType.Xz, info.Detection.OuterCompressionType); + Assert.NotNull(info.EntryCount); + } + + [Fact] + public void InspectArchive_MultiVolumeRar_ReturnsVolumeMetadata() + { + FileInfo[] parts = + [ + new(GetTestArchivePath("Rar.multi.solid.part01.rar")), + new(GetTestArchivePath("Rar.multi.solid.part02.rar")), + new(GetTestArchivePath("Rar.multi.solid.part03.rar")), + new(GetTestArchivePath("Rar.multi.solid.part04.rar")), + new(GetTestArchivePath("Rar.multi.solid.part05.rar")), + new(GetTestArchivePath("Rar.multi.solid.part06.rar")), + ]; + + var info = ArchiveFactory.InspectArchive(parts); + + Assert.NotNull(info); + Assert.Equal(ArchiveType.Rar, info.Detection.ContainerType); + Assert.Equal(parts.Length, info.PhysicalPartCount); + Assert.True(info.IsMultiVolume); + } + + [Fact] + public void InspectArchive_MultiVolumeRarPath_DiscoversAllParts() + { + var info = ArchiveFactory.InspectArchive(GetTestArchivePath("Rar.multi.solid.part01.rar")); + + Assert.NotNull(info); + Assert.Equal(6, info.PhysicalPartCount); + Assert.True(info.IsMultiVolume); + } + private MemoryStream CreatePrefixedArchiveStream(string archiveName, int prefixLength) { var archiveBytes = File.ReadAllBytes(GetTestArchivePath(archiveName)); diff --git a/tests/SharpCompress.Test/BZip2/BZip2StreamAsyncTests.cs b/tests/SharpCompress.Test/BZip2/BZip2StreamAsyncTests.cs index d5b2ef78..fc7e649c 100644 --- a/tests/SharpCompress.Test/BZip2/BZip2StreamAsyncTests.cs +++ b/tests/SharpCompress.Test/BZip2/BZip2StreamAsyncTests.cs @@ -151,19 +151,8 @@ public class BZip2StreamAsyncTests Assert.True(compressed.Length > 0); // Decompress and verify -#if LEGACY_DOTNET + // MemoryStream has nothing to dispose asynchronously using (var readStream = new MemoryStream(compressed)) - { - using ( - var bzip2Stream = await BZip2Stream.CreateAsync( - new AsyncOnlyStream(readStream), - SharpCompress.Compressors.CompressionMode.Decompress, - false - ) - ) - { -#else - await using (var readStream = new MemoryStream(compressed)) { await using ( var bzip2Stream = await BZip2Stream.CreateAsync( @@ -173,7 +162,6 @@ public class BZip2StreamAsyncTests ) ) { -#endif var result = new StringBuilder(); var buffer = new byte[256]; int bytesRead; diff --git a/tests/SharpCompress.Test/GZip/GZipArchiveAsyncTests.cs b/tests/SharpCompress.Test/GZip/GZipArchiveAsyncTests.cs index d7812b57..ba5a7782 100644 --- a/tests/SharpCompress.Test/GZip/GZipArchiveAsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipArchiveAsyncTests.cs @@ -240,4 +240,35 @@ public class GZipArchiveAsyncTests : ArchiveTests Assert.True(entryStream.IsDisposed); } + + [Fact] + public async ValueTask GZip_Archive_SaveTo_Async_Preserves_Modification_Time() + { + var modificationTime = new DateTime(2024, 1, 2, 3, 4, 5); + using var destination = new MemoryStream(); + await using (var archive = (GZipArchive)await GZipArchive.CreateAsyncArchive()) + { + await archive.AddEntryAsync( + "contents.bin", + new MemoryStream(new byte[] { 1, 2, 3, 4 }), + size: 4, + modified: modificationTime + ); + await archive.SaveToAsync(destination, new GZipWriterOptions()); + } + + destination.Position = 0; + await using var result = await GZipArchive.OpenAsyncArchive(destination); + var entry = await result.EntriesAsync.FirstAsync(); + + Assert.Equal(modificationTime, entry.LastModifiedTime); + } + + [Fact] + public async ValueTask GZip_Archive_IsGZipFileAsync_ReturnsFalse_For_Truncated_Header() + { + using var stream = new MemoryStream(new byte[] { 0x1F, 0x8B, 8 }); + + Assert.False(await GZipArchive.IsGZipFileAsync(stream)); + } } diff --git a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs index 2e98c962..acabcb4b 100644 --- a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs @@ -68,11 +68,8 @@ public class GZipCrcExtractionTests : TestBase [Fact] public async Task GZipArchive_WriteToFileAsync_Throws_On_Crc_Mismatch() { -#if LEGACY_DOTNET + // MemoryStream has nothing to dispose asynchronously using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); -#else - await using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); -#endif await using var archive = await GZipArchive.OpenAsyncArchive(stream); var entry = await archive.EntriesAsync.SingleAsync(); var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); diff --git a/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs index 83d2b352..98914780 100644 --- a/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs @@ -1,4 +1,5 @@ using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.IO; @@ -37,4 +38,33 @@ public class GZipReaderAsyncTests : ReaderTests } } } + + [Fact] + public async ValueTask GZip_ReaderFactory_FlatGZip_Async() + { + var source = new byte[2048]; + for (var i = 0; i < source.Length; i++) + { + source[i] = 0xFF; + } + + var gzipPath = Path.Combine(SCRATCH_FILES_PATH, "Flat.bin.gz"); + using (var output = File.Create(gzipPath)) + using (var gzip = new GZipStream(output, CompressionMode.Compress)) + { + await gzip.WriteAsync(source, 0, source.Length); + } + + using Stream stream = File.OpenRead(gzipPath); + await using var reader = await ReaderFactory.OpenAsyncReader(stream); + Assert.IsType(reader); + Assert.True(await reader.MoveToNextEntryAsync()); + + using var ms = new MemoryStream(); + await reader.WriteEntryToAsync(ms); + Assert.Equal(source.Length, ms.Length); + Assert.Equal(source, ms.ToArray()); + + Assert.False(await reader.MoveToNextEntryAsync()); + } } diff --git a/tests/SharpCompress.Test/GZip/GZipReaderTests.cs b/tests/SharpCompress.Test/GZip/GZipReaderTests.cs index fb3936a8..ab870da2 100644 --- a/tests/SharpCompress.Test/GZip/GZipReaderTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipReaderTests.cs @@ -48,6 +48,12 @@ public class GZipReaderTests : ReaderTests using var reader = ReaderFactory.OpenReader(stream); Assert.IsType(reader); Assert.True(reader.MoveToNextEntry()); + + using var ms = new MemoryStream(); + reader.WriteEntryTo(ms); + Assert.Equal(source.Length, ms.Length); + Assert.Equal(source, ms.ToArray()); + Assert.False(reader.MoveToNextEntry()); } } diff --git a/tests/SharpCompress.Test/GZip/GZipWriterAsyncTests.cs b/tests/SharpCompress.Test/GZip/GZipWriterAsyncTests.cs index ba9ecd61..56136e8e 100644 --- a/tests/SharpCompress.Test/GZip/GZipWriterAsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipWriterAsyncTests.cs @@ -1,6 +1,8 @@ using System.IO; using System.Threading.Tasks; using SharpCompress.Common; +using SharpCompress.Providers; +using SharpCompress.Providers.System; using SharpCompress.Test.Mocks; using SharpCompress.Writers; using SharpCompress.Writers.GZip; @@ -91,4 +93,33 @@ public class GZipWriterAsyncTests : WriterTests Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz") ); } + + [Fact] + public async ValueTask GZip_Writer_Async_With_System_Provider() + { + var contents = new byte[] { 1, 2, 3, 4 }; + var providers = CompressionProviderRegistry.Default.With( + new SystemGZipCompressionProvider() + ); + using var source = new MemoryStream(contents); + using var destination = new MemoryStream(); + await using ( + var writer = new GZipWriter( + destination, + new GZipWriterOptions { LeaveStreamOpen = true, Providers = providers } + ) + ) + { + await writer.WriteAsync("contents.bin", source); + } + + destination.Position = 0; + using var compressed = new SystemGZipCompressionProvider().CreateDecompressStream( + destination + ); + using var actual = new MemoryStream(); + compressed.CopyTo(actual); + + Assert.Equal(contents, actual.ToArray()); + } } diff --git a/tests/SharpCompress.Test/GZip/GZipWriterDirectoryTests.cs b/tests/SharpCompress.Test/GZip/GZipWriterDirectoryTests.cs index 5d51c441..9941d256 100644 --- a/tests/SharpCompress.Test/GZip/GZipWriterDirectoryTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipWriterDirectoryTests.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Writers.GZip; using Xunit; @@ -16,4 +17,15 @@ public class GZipWriterDirectoryTests : TestBase Assert.Throws(() => writer.WriteDirectory("test-dir", DateTime.Now)); } + + [Fact] + public async ValueTask GZipWriter_WriteDirectoryAsync_ThrowsNotSupportedException() + { + using var memoryStream = new MemoryStream(); + using var writer = new GZipWriter(memoryStream, new GZipWriterOptions()); + + await Assert.ThrowsAsync(async () => + await writer.WriteDirectoryAsync("test-dir", DateTime.Now) + ); + } } diff --git a/tests/SharpCompress.Test/LargeArchiveTests.cs b/tests/SharpCompress.Test/LargeArchiveTests.cs new file mode 100644 index 00000000..84fb0f7f --- /dev/null +++ b/tests/SharpCompress.Test/LargeArchiveTests.cs @@ -0,0 +1,208 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Crypto; +using SharpCompress.Readers; +using SharpCompress.Test.Mocks; +using Xunit; + +namespace SharpCompress.Test; + +[Collection(LargeArchiveCollection.Name)] +public class LargeArchiveTests : TestBase +{ + private const long LargeFileSize = 64L * 1024 * 1024; + private const uint LargeFileCrc = 0xF9081EB0; + private const int BufferSize = 64 * 1024; + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.7z")] + public void OpenArchive_ShouldStreamLargeEntry(string fixtureName) + { + using var stream = File.OpenRead(GetMaterializedFixturePath(fixtureName)); + using var archive = ArchiveFactory.OpenArchive(stream); + + VerifyArchive(archive); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.7z")] + public async Task OpenAsyncArchive_ShouldStreamLargeEntry(string fixtureName) + { + await using var stream = new AsyncOnlyStream( + File.OpenRead(await GetMaterializedFixturePathAsync(fixtureName)) + ); + await using var archive = await ArchiveFactory.OpenAsyncArchive(stream); + + var entry = await GetSingleEntryAsync(archive); + + var entryStream = await entry.OpenEntryStreamAsync(); + await using var entryStreamScope = entryStream.DisposeAsyncScope(); + await VerifyContentAsync(entry.Key, entryStream); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.tar.gz")] + public void OpenReader_ShouldStreamLargeEntry(string fixtureName) + { + using var stream = File.OpenRead(GetMaterializedFixturePath(fixtureName)); + using var reader = ReaderFactory.OpenReader(stream); + + VerifyReader(reader); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.tar.gz")] + public async Task OpenAsyncReader_ShouldStreamLargeEntry(string fixtureName) + { + await using var stream = new AsyncOnlyStream( + File.OpenRead(await GetMaterializedFixturePathAsync(fixtureName)) + ); + await using var reader = await ReaderFactory.OpenAsyncReader(stream); + + Assert.True(await reader.MoveToNextEntryAsync()); + Assert.False(reader.Entry.IsDirectory); + + await using var entryStream = await reader.OpenEntryStreamAsync(); + await VerifyContentAsync(reader.Entry.Key, entryStream); + Assert.False(await reader.MoveToNextEntryAsync()); + } + + [Fact] + public async Task OpenAsyncReader_WithGZipExtensionHint_ShouldStreamLargeTarEntry() + { + using var file = File.OpenRead(GetFixturePath("Large/Large.tar.gz")); + await using var stream = new AsyncOnlyStream(new ForwardOnlyStream(file)); + var options = new ReaderOptions { ExtensionHint = "gz" }; + + await using var reader = await ReaderFactory.OpenAsyncReader(stream, options); + + Assert.True(await reader.MoveToNextEntryAsync()); + Assert.False(reader.Entry.IsDirectory); + await using var entryStream = await reader.OpenEntryStreamAsync(); + await VerifyContentAsync(reader.Entry.Key, entryStream); + Assert.False(await reader.MoveToNextEntryAsync()); + } + + private static string GetFixturePath(string fixtureName) => + Path.Combine(TEST_ARCHIVES_PATH, fixtureName); + + private string GetMaterializedFixturePath(string fixtureName) => + fixtureName == "Large/Large.tar" ? MaterializeTarFixture() : GetFixturePath(fixtureName); + + private async Task GetMaterializedFixturePathAsync(string fixtureName) => + fixtureName == "Large/Large.tar" + ? await MaterializeTarFixtureAsync() + : GetFixturePath(fixtureName); + + private string MaterializeTarFixture() + { + var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar"); + using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz")); + using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress); + using var tarStream = File.Create(tarPath); + gzipStream.CopyTo(tarStream); + return tarPath; + } + + private async Task MaterializeTarFixtureAsync() + { + var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar"); + using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz")); + using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress); + using var tarStream = File.Create(tarPath); + await gzipStream.CopyToAsync(tarStream); + return tarPath; + } + + private static void VerifyArchive(IArchive archive) + { + var entry = Assert.Single(archive.Entries); + + Assert.False(entry.IsDirectory); + using var entryStream = entry.OpenEntryStream(); + VerifyContent(entry.Key, entryStream); + } + + private static async Task GetSingleEntryAsync(IAsyncArchive archive) + { + IArchiveEntry? entry = null; + await foreach (var candidate in archive.EntriesAsync) + { + Assert.Null(entry); + entry = candidate; + } + + return entry ?? throw new InvalidOperationException("The archive contains no entries."); + } + + private static void VerifyReader(IReader reader) + { + Assert.True(reader.MoveToNextEntry()); + Assert.False(reader.Entry.IsDirectory); + using var entryStream = reader.OpenEntryStream(); + VerifyContent(reader.Entry.Key, entryStream); + Assert.False(reader.MoveToNextEntry()); + } + + private static void VerifyContent(string? key, Stream entryStream) + { + Assert.Equal("large.bin", key); + + using var crcStream = new Crc32Stream(Stream.Null); + var buffer = new byte[BufferSize]; + long length = 0; + int bytesRead; + while ((bytesRead = entryStream.Read(buffer, 0, buffer.Length)) > 0) + { + crcStream.Write(buffer, 0, bytesRead); + length += bytesRead; + } + + Assert.Equal(LargeFileSize, length); + Assert.Equal(LargeFileCrc, crcStream.Crc); + } + + private static async Task VerifyContentAsync(string? key, Stream entryStream) + { + Assert.Equal("large.bin", key); + + using var crcStream = new Crc32Stream(Stream.Null); + var buffer = new byte[BufferSize]; + long length = 0; + int bytesRead; + while ((bytesRead = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + await crcStream.WriteAsync(buffer, 0, bytesRead); + length += bytesRead; + } + + Assert.Equal(LargeFileSize, length); + Assert.Equal(LargeFileCrc, crcStream.Crc); + } +} + +[CollectionDefinition(LargeArchiveCollection.Name, DisableParallelization = true)] +public sealed class LargeArchiveCollection +{ + public const string Name = "Large archive fixtures"; +} diff --git a/tests/SharpCompress.Test/Mocks/AsyncOnlyStream.cs b/tests/SharpCompress.Test/Mocks/AsyncOnlyStream.cs index 232f60ee..9dec5e5a 100644 --- a/tests/SharpCompress.Test/Mocks/AsyncOnlyStream.cs +++ b/tests/SharpCompress.Test/Mocks/AsyncOnlyStream.cs @@ -2,10 +2,11 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; -public class AsyncOnlyStream(Stream stream, bool disposeStream = true) : Stream +public class AsyncOnlyStream(Stream stream, bool disposeStream = true) : AsyncDisposableStream { private readonly Stream _stream = stream ?? throw new ArgumentNullException(nameof(stream)); diff --git a/tests/SharpCompress.Test/Mocks/FlushOnDisposeStream.cs b/tests/SharpCompress.Test/Mocks/FlushOnDisposeStream.cs index 63f20702..28dff313 100644 --- a/tests/SharpCompress.Test/Mocks/FlushOnDisposeStream.cs +++ b/tests/SharpCompress.Test/Mocks/FlushOnDisposeStream.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; @@ -8,7 +9,7 @@ namespace SharpCompress.Test.Mocks; // CryptoStream doesn't always trigger the Flush, so this class is used instead // See https://referencesource.microsoft.com/#mscorlib/system/security/cryptography/cryptostream.cs,141 -public class FlushOnDisposeStream(Stream innerStream) : Stream +public class FlushOnDisposeStream(Stream innerStream) : AsyncDisposableStream { public override bool CanRead => innerStream.CanRead; @@ -48,12 +49,10 @@ public class FlushOnDisposeStream(Stream innerStream) : Stream base.Dispose(disposing); } -#if !LEGACY_DOTNET public override async ValueTask DisposeAsync() { await innerStream.FlushAsync(); innerStream.Close(); await base.DisposeAsync(); } -#endif } diff --git a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs index 32961ebc..f6b7ec8e 100644 --- a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs +++ b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; @@ -9,7 +10,7 @@ namespace SharpCompress.Test.Mocks; /// A forward-only stream wrapper that delegates directly to the underlying stream /// without any buffering. Supports reading and writing but not seeking. /// -public class ForwardOnlyStream : Stream +public class ForwardOnlyStream : AsyncDisposableStream { private readonly Stream _stream; private bool _isDisposed; @@ -142,17 +143,15 @@ public class ForwardOnlyStream : Stream } } -#if !LEGACY_DOTNET public override async ValueTask DisposeAsync() { if (!_isDisposed) { - await _stream.DisposeAsync(); + await _stream.DisposeAsyncCompat(); _isDisposed = true; } await base.DisposeAsync(); } -#endif private void ThrowIfDisposed() { diff --git a/tests/SharpCompress.Test/Mocks/TestStream.cs b/tests/SharpCompress.Test/Mocks/TestStream.cs index d3ff48a8..5061afc3 100644 --- a/tests/SharpCompress.Test/Mocks/TestStream.cs +++ b/tests/SharpCompress.Test/Mocks/TestStream.cs @@ -2,10 +2,11 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; -public class TestStream(Stream stream, bool read, bool write, bool seek) : Stream +public class TestStream(Stream stream, bool read, bool write, bool seek) : AsyncDisposableStream { public TestStream(Stream stream) : this(stream, stream.CanRead, stream.CanWrite, stream.CanSeek) { } @@ -50,14 +51,14 @@ public class TestStream(Stream stream, bool read, bool write, bool seek) : Strea Memory buffer, CancellationToken cancellationToken = default ) => stream.ReadAsync(buffer, cancellationToken); +#endif public override async ValueTask DisposeAsync() { await base.DisposeAsync(); - await stream.DisposeAsync(); + await stream.DisposeAsyncCompat(); IsDisposed = true; } -#endif public override long Seek(long offset, SeekOrigin origin) => stream.Seek(offset, origin); diff --git a/tests/SharpCompress.Test/Mocks/ThrowOnFlushStream.cs b/tests/SharpCompress.Test/Mocks/ThrowOnFlushStream.cs index 2cf1a84c..911762a5 100644 --- a/tests/SharpCompress.Test/Mocks/ThrowOnFlushStream.cs +++ b/tests/SharpCompress.Test/Mocks/ThrowOnFlushStream.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; @@ -9,7 +10,7 @@ namespace SharpCompress.Test.Mocks; /// A stream wrapper that throws NotSupportedException on Flush() calls. /// This is used to test that archive iteration handles streams that don't support flushing. /// -public class ThrowOnFlushStream : Stream +public class ThrowOnFlushStream : AsyncDisposableStream { private readonly Stream inner; diff --git a/tests/SharpCompress.Test/Mocks/TruncatedStream.cs b/tests/SharpCompress.Test/Mocks/TruncatedStream.cs index 8699a430..55037f2e 100644 --- a/tests/SharpCompress.Test/Mocks/TruncatedStream.cs +++ b/tests/SharpCompress.Test/Mocks/TruncatedStream.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using SharpCompress.IO; namespace SharpCompress.Test.Mocks; @@ -7,7 +8,7 @@ namespace SharpCompress.Test.Mocks; /// A stream wrapper that truncates the underlying stream after reading a specified number of bytes. /// Used for testing error handling when streams end prematurely. /// -public class TruncatedStream : Stream +public class TruncatedStream : AsyncDisposableStream { private readonly Stream baseStream; private readonly long truncateAfterBytes; diff --git a/tests/SharpCompress.Test/OptionsUsabilityTests.cs b/tests/SharpCompress.Test/OptionsUsabilityTests.cs index 1902e4ab..0226aed8 100644 --- a/tests/SharpCompress.Test/OptionsUsabilityTests.cs +++ b/tests/SharpCompress.Test/OptionsUsabilityTests.cs @@ -220,6 +220,36 @@ public class OptionsUsabilityTests : TestBase Assert.Equal(19, source.CopyBufferSize); } + [Fact] + public async Task GZipWriter_Uses_WriterOptions_BufferSize_Async() + { + await using var source = new TrackingReadStream(new byte[100]); + await using var destination = new MemoryStream(); + await using var writer = new GZipWriter( + destination, + new GZipWriterOptions { BufferSize = 23 } + ); + + await writer.WriteAsync("buffer-size.txt", source, DateTime.Now); + + Assert.Equal(23, source.CopyBufferSize); + } + + [Fact] + public async Task ZipWriter_Uses_WriterOptions_BufferSize_Async() + { + await using var source = new TrackingReadStream(new byte[100]); + await using var destination = new MemoryStream(); + await using var writer = new ZipWriter( + destination, + new ZipWriterOptions(CompressionType.None) { BufferSize = 29 } + ); + + await writer.WriteAsync("buffer-size.txt", source, DateTime.Now); + + Assert.Equal(29, source.CopyBufferSize); + } + [Fact] public void TarWriter_Uses_WriterOptions_BufferSize_ForTransfer() { diff --git a/tests/SharpCompress.Test/ReaderTests.cs b/tests/SharpCompress.Test/ReaderTests.cs index 4c9024c9..64469219 100644 --- a/tests/SharpCompress.Test/ReaderTests.cs +++ b/tests/SharpCompress.Test/ReaderTests.cs @@ -159,18 +159,12 @@ public abstract class ReaderTests : TestBase { using var file = File.OpenRead(testArchive); -#if !LEGACY_DOTNET - await using var protectedStream = SharpCompressStream.CreateNonDisposing( + // SharpCompressStream is not yet an AsyncDisposableStream, so scope its disposal + var protectedStream = SharpCompressStream.CreateNonDisposing( new ForwardOnlyStream(file, options.BufferSize) ); + await using var protectedStreamScope = protectedStream.DisposeAsyncScope(); await using var testStream = new TestStream(protectedStream); -#else - - using var protectedStream = SharpCompressStream.CreateNonDisposing( - new ForwardOnlyStream(file, options.BufferSize) - ); - using var testStream = new TestStream(protectedStream); -#endif await using ( var reader = await ReaderFactory.OpenAsyncReader( new AsyncOnlyStream(testStream), diff --git a/tests/SharpCompress.Test/Security/SymbolicLinkExtractionTests.cs b/tests/SharpCompress.Test/Security/SymbolicLinkExtractionTests.cs new file mode 100644 index 00000000..a84c3267 --- /dev/null +++ b/tests/SharpCompress.Test/Security/SymbolicLinkExtractionTests.cs @@ -0,0 +1,336 @@ +#if NET8_0_OR_GREATER +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Common; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test.Security; + +public class SymbolicLinkExtractionTests : TestBase +{ + private const int TarBlockSize = 512; + + [Theory] + [InlineData("ReaderAll")] + [InlineData("ReaderEntry")] + [InlineData("Archive")] + [InlineData("ArchiveEntry")] + [InlineData("AsyncReaderAll")] + [InlineData("AsyncReaderEntry")] + [InlineData("AsyncArchive")] + [InlineData("AsyncArchiveEntry")] + public async Task SymbolicLinkTargetOutsideDestination_ShouldThrowBeforeInvokingHandler( + string api + ) + { + var destinationDirectory = GetScratchPath("extract"); + var outsideDirectory = GetScratchPath("outside"); + var archivePath = GetScratch2Path("symbolic-link-outside.tar"); + Directory.CreateDirectory(destinationDirectory); + Directory.CreateDirectory(outsideDirectory); + BuildTar(archivePath, "../outside"); + + var handlerCalls = 0; + var options = new ExtractionOptions { SymbolicLinkHandler = (_, _) => handlerCalls++ }; + + var exception = await ExtractAsync(api, archivePath, destinationDirectory, options); + + var extractionException = Assert.IsType(exception); + Assert.Contains("symbolic link whose target is outside", extractionException.Message); + Assert.Equal(0, handlerCalls); + Assert.False(File.Exists(Path.Combine(outsideDirectory, "secret.txt"))); + } + + [Theory] + [InlineData("ReaderAll")] + [InlineData("Archive")] + [InlineData("AsyncReaderAll")] + [InlineData("AsyncArchive")] + public async Task EntriesBeneathSymbolicLink_ShouldNotBeExtracted(string api) + { + var destinationDirectory = GetScratchPath("extract"); + var targetDirectory = Path.Combine(destinationDirectory, "target"); + var archivePath = GetScratch2Path("symbolic-link-inside.tar"); + Directory.CreateDirectory(destinationDirectory); + Directory.CreateDirectory(targetDirectory); + BuildTar(archivePath, "target"); + + var handlerCalls = 0; + var options = new ExtractionOptions + { + SymbolicLinkHandler = (linkPath, linkTarget) => + { + handlerCalls++; + CreateReparsePoint(linkPath, linkTarget); + }, + }; + + var extractionException = ( + await ExtractAsync(api, archivePath, destinationDirectory, options) + ).NotNull(); + + Assert.Contains("symbolic link or reparse point", extractionException.ToString()); + Assert.Equal(1, handlerCalls); + Assert.False(File.Exists(Path.Combine(targetDirectory, "secret.txt"))); + } + + private static void CreateReparsePoint(string linkPath, string linkTarget) + { + if (OperatingSystem.IsWindows()) + { + // Directory junctions are reparse points that need no elevation, unlike symbolic links. + // Junction targets must be absolute. + var absoluteTarget = Path.GetFullPath( + Path.Combine(Path.GetDirectoryName(linkPath).NotNull(), linkTarget) + ); + var startInfo = new ProcessStartInfo("cmd.exe") + { + UseShellExecute = false, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("/c"); + startInfo.ArgumentList.Add("mklink"); + startInfo.ArgumentList.Add("/j"); + startInfo.ArgumentList.Add(linkPath); + startInfo.ArgumentList.Add(absoluteTarget); + + using var process = Process.Start(startInfo).NotNull(); + process.WaitForExit(); + Assert.Equal(0, process.ExitCode); + } + else + { + Directory.CreateSymbolicLink(linkPath, linkTarget); + } + } + + private static void BuildTar(string path, string linkTarget) + { + using var stream = File.Create(path); + WriteTarEntry(stream, "link", (byte)'2', linkTarget, Array.Empty()); + WriteTarEntry(stream, "link/secret.txt", (byte)'0', null, Encoding.UTF8.GetBytes("secret")); + stream.Write(new byte[TarBlockSize * 2]); + } + + private static void WriteTarEntry( + Stream stream, + string name, + byte entryType, + string? linkTarget, + byte[] data + ) + { + var header = new byte[TarBlockSize]; + WriteString(header, 0, 100, name); + WriteOctal(header, 100, 8, 0b110_100_100); + WriteOctal(header, 108, 8, 0); + WriteOctal(header, 116, 8, 0); + WriteOctal(header, 124, 12, data.Length); + WriteOctal(header, 136, 12, 0); + Array.Fill(header, (byte)' ', 148, 8); + header[156] = entryType; + WriteString(header, 157, 100, linkTarget ?? string.Empty); + WriteString(header, 257, 6, "ustar"); + WriteString(header, 263, 2, "00"); + + var checksum = header.Sum(value => value); + WriteString(header, 148, 6, Convert.ToString(checksum, 8).PadLeft(6, '0')); + header[154] = 0; + header[155] = (byte)' '; + + stream.Write(header); + stream.Write(data); + + var padding = (TarBlockSize - (data.Length % TarBlockSize)) % TarBlockSize; + if (padding > 0) + { + stream.Write(new byte[padding]); + } + } + + private static void WriteString(byte[] buffer, int offset, int length, string value) + { + var bytes = Encoding.ASCII.GetBytes(value); + Array.Copy(bytes, 0, buffer, offset, Math.Min(bytes.Length, length)); + } + + private static void WriteOctal(byte[] buffer, int offset, int length, long value) + { + WriteString( + buffer, + offset, + length - 1, + Convert.ToString(value, 8).PadLeft(length - 1, '0') + ); + buffer[offset + length - 1] = 0; + } + + private static Task ExtractAsync( + string api, + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) => + api switch + { + "ReaderAll" => Task.FromResult( + RecordException(() => + ExtractWithReaderAll(archivePath, destinationDirectory, options) + ) + ), + "ReaderEntry" => Task.FromResult( + RecordException(() => + ExtractWithReaderEntry(archivePath, destinationDirectory, options) + ) + ), + "Archive" => Task.FromResult( + RecordException(() => + ExtractWithArchive(archivePath, destinationDirectory, options) + ) + ), + "ArchiveEntry" => Task.FromResult( + RecordException(() => + ExtractWithArchiveEntry(archivePath, destinationDirectory, options) + ) + ), + "AsyncReaderAll" => RecordExceptionAsync(() => + ExtractWithAsyncReaderAllAsync(archivePath, destinationDirectory, options) + ), + "AsyncReaderEntry" => RecordExceptionAsync(() => + ExtractWithAsyncReaderEntryAsync(archivePath, destinationDirectory, options) + ), + "AsyncArchive" => RecordExceptionAsync(() => + ExtractWithAsyncArchiveAsync(archivePath, destinationDirectory, options) + ), + "AsyncArchiveEntry" => RecordExceptionAsync(() => + ExtractWithAsyncArchiveEntryAsync(archivePath, destinationDirectory, options) + ), + _ => throw new ArgumentOutOfRangeException(nameof(api), api, null), + }; + + private static Exception? RecordException(Action action) + { + try + { + action(); + return null; + } + catch (Exception exception) + { + return exception; + } + } + + private static async Task RecordExceptionAsync(Func action) + { + try + { + await action(); + return null; + } + catch (Exception exception) + { + return exception; + } + } + + private static void ExtractWithReaderAll( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var stream = File.OpenRead(archivePath); + using var reader = ReaderFactory.OpenReader(stream); + reader.WriteAllToDirectory(destinationDirectory, options); + } + + private static void ExtractWithReaderEntry( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var stream = File.OpenRead(archivePath); + using var reader = ReaderFactory.OpenReader(stream); + Assert.True(reader.MoveToNextEntry()); + reader.WriteEntryToDirectory(destinationDirectory, options); + } + + private static void ExtractWithArchive( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var archive = ArchiveFactory.OpenArchive(archivePath); + archive.WriteToDirectory(destinationDirectory, options); + } + + private static void ExtractWithArchiveEntry( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var archive = ArchiveFactory.OpenArchive(archivePath); + archive.Entries.First().WriteToDirectory(destinationDirectory, options); + } + + private static async Task ExtractWithAsyncReaderAllAsync( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var stream = File.OpenRead(archivePath); + await using var reader = await ReaderFactory.OpenAsyncReader(stream); + await reader.WriteAllToDirectoryAsync(destinationDirectory, options); + } + + private static async Task ExtractWithAsyncReaderEntryAsync( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + using var stream = File.OpenRead(archivePath); + await using var reader = await ReaderFactory.OpenAsyncReader(stream); + Assert.True(await reader.MoveToNextEntryAsync()); + await reader.WriteEntryToDirectoryAsync(destinationDirectory, options); + } + + private static async Task ExtractWithAsyncArchiveAsync( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + await using var archive = await ArchiveFactory.OpenAsyncArchive(archivePath); + await archive.WriteToDirectoryAsync(destinationDirectory, options); + } + + private static async Task ExtractWithAsyncArchiveEntryAsync( + string archivePath, + string destinationDirectory, + ExtractionOptions options + ) + { + await using var archive = await ArchiveFactory.OpenAsyncArchive(archivePath); + + await foreach (var entry in archive.EntriesAsync) + { + await entry.WriteToDirectoryAsync(destinationDirectory, options); + return; + } + + throw new InvalidOperationException("Archive did not contain an entry."); + } +} +#endif diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveParallelTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveParallelTests.cs new file mode 100644 index 00000000..bdfb5b67 --- /dev/null +++ b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveParallelTests.cs @@ -0,0 +1,310 @@ +using System.IO; +using System.Linq; +using SharpCompress.Archives; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Common; +using SharpCompress.Common.SevenZip; +using SharpCompress.Factories; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test.SevenZip; + +// Parallel counterpart of SevenZipArchiveTests: every test here opts in via +// ReaderOptions.EnableParallelism (default is sequential decode), covering the same archives to +// verify decoded content is identical regardless of which decode path 7-Zip's automatic parallel +// LZMA2 solid-folder decode takes. +public class SevenZipArchiveParallelTests : ArchiveTests +{ + private static ReaderOptions Parallel(ReaderOptions options) => + options.WithEnableParallelism(true); + + [Fact] + public void SevenZipArchive_Solid_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.solid.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_NonSolid_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.nonsolid.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_LZMA_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.LZMA.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_LZMA_PathRead_Parallel() => + ArchiveFileRead("7Zip.LZMA.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_LZMAAES_StreamRead_Parallel() => + ArchiveStreamRead( + "7Zip.LZMA.Aes.7z", + Parallel(ReaderOptions.ForExternalStream) with + { + Password = "testpassword", + } + ); + + [Fact] + public void SevenZipArchive_LZMAAES_PathRead_Parallel() => + ArchiveFileRead( + "7Zip.LZMA.Aes.7z", + Parallel(ReaderOptions.ForFilePath) with + { + Password = "testpassword", + } + ); + + [Fact] + public void SevenZipArchive_PPMd_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.PPMd.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_PPMd_PathRead_Parallel() => + ArchiveFileRead("7Zip.PPMd.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_LZMA2_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.LZMA2.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_LZMA2_PathRead_Parallel() => + ArchiveFileRead("7Zip.LZMA2.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_LZMA2_EXE_StreamRead_Parallel() => + ArchiveStreamRead( + new SevenZipFactory(), + "7Zip.LZMA2.exe", + Parallel(ReaderOptions.ForExternalStream).WithLookForHeader(true) + ); + + [Fact] + public void SevenZipArchive_LZMA2_EXE_PathRead_Parallel() => + ArchiveFileRead( + "7Zip.LZMA2.exe", + Parallel(ReaderOptions.ForFilePath).WithLookForHeader(true), + new SevenZipFactory() + ); + + [Fact] + public void SevenZipArchive_LZMA2AES_StreamRead_Parallel() => + ArchiveStreamRead( + "7Zip.LZMA2.Aes.7z", + Parallel(ReaderOptions.ForExternalStream) with + { + Password = "testpassword", + } + ); + + [Fact] + public void SevenZipArchive_LZMA2AES_PathRead_Parallel() => + ArchiveFileRead( + "7Zip.LZMA2.Aes.7z", + Parallel(ReaderOptions.ForFilePath) with + { + Password = "testpassword", + } + ); + + [Fact] + public void SevenZipArchive_BZip2_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.BZip2.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_BZip2_PathRead_Parallel() => + ArchiveFileRead("7Zip.BZip2.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_Copy_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.Copy.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_Copy_PathRead_Parallel() => + ArchiveFileRead("7Zip.Copy.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_Copy_CompressionType_Parallel() + { + using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "7Zip.Copy.7z"))) + using ( + var archive = SevenZipArchive.OpenArchive( + stream, + Parallel(ReaderOptions.ForExternalStream) + ) + ) + { + foreach (var entry in archive.Entries.Where(e => !e.IsDirectory)) + { + Assert.Equal(CompressionType.None, entry.CompressionType); + } + } + } + + [Fact] + public void SevenZipArchive_ZSTD_StreamRead_Parallel() => + ArchiveStreamRead("7Zip.ZSTD.7z", Parallel(ReaderOptions.ForExternalStream)); + + [Fact] + public void SevenZipArchive_ZSTD_PathRead_Parallel() => + ArchiveFileRead("7Zip.ZSTD.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_ZSTD_Split_Parallel() => + ArchiveStreamMultiRead( + Parallel(ReaderOptions.ForFilePath), + "7Zip.ZSTD.Split.7z.001", + "7Zip.ZSTD.Split.7z.002", + "7Zip.ZSTD.Split.7z.003", + "7Zip.ZSTD.Split.7z.004", + "7Zip.ZSTD.Split.7z.005", + "7Zip.ZSTD.Split.7z.006" + ); + + [Fact] + public void SevenZipArchive_EOS_FileRead_Parallel() => + ArchiveFileRead("7Zip.eos.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_Delta_FileRead_Parallel() => + ArchiveFileRead("7Zip.delta.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_ARM_FileRead_Parallel() => + ArchiveFileRead("7Zip.ARM.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_ARMT_FileRead_Parallel() => + ArchiveFileRead("7Zip.ARMT.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_BCJ_FileRead_Parallel() => + ArchiveFileRead("7Zip.BCJ.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_BCJ2_FileRead_Parallel() => + ArchiveFileRead("7Zip.BCJ2.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_IA64_FileRead_Parallel() => + ArchiveFileRead("7Zip.IA64.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_PPC_FileRead_Parallel() => + ArchiveFileRead("7Zip.PPC.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_SPARC_FileRead_Parallel() => + ArchiveFileRead("7Zip.SPARC.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_ARM64_FileRead_Parallel() => + ArchiveFileRead("7Zip.ARM64.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_RISCV_FileRead_Parallel() => + ArchiveFileRead("7Zip.RISCV.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_Filters_FileRead_Parallel() => + ArchiveFileRead("7Zip.Filters.7z", Parallel(ReaderOptions.ForFilePath)); + + [Fact] + public void SevenZipArchive_Tar_PathRead_Parallel() + { + using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "7Zip.Tar.tar.7z"))) + using ( + var archive = SevenZipArchive.OpenArchive( + stream, + Parallel(ReaderOptions.ForExternalStream) + ) + ) + { + var entry = archive.Entries.First(); + entry.WriteToFile(Path.Combine(SCRATCH_FILES_PATH, entry.Key.NotNull())); + + var size = entry.Size; + var scratch = new FileInfo(Path.Combine(SCRATCH_FILES_PATH, "7Zip.Tar.tar")); + var test = new FileInfo(Path.Combine(TEST_ARCHIVES_PATH, "7Zip.Tar.tar")); + + Assert.Equal(size, scratch.Length); + Assert.Equal(size, test.Length); + } + + CompareArchivesByPath( + Path.Combine(SCRATCH_FILES_PATH, "7Zip.Tar.tar"), + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.Tar.tar") + ); + } + + [Fact] + public void SevenZipArchive_Solid_ExtractAllEntries_Contiguous_Parallel() + { + // This test verifies that solid archives iterate entries as contiguous streams + // rather than recreating the decompression stream for each entry, with the parallel + // decode path opted in. + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.7z"); + using var archive = SevenZipArchive.OpenArchive( + testArchive, + Parallel(ReaderOptions.ForFilePath) + ); + Assert.True(archive.IsSolid); + + using var reader = archive.ExtractAllEntries(); + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + reader.WriteEntryToDirectory(SCRATCH_FILES_PATH); + } + } + + VerifyFiles(); + } + + [Fact] + public void SevenZipArchive_EmptyStream_WriteToDirectory_Parallel() + { + // This test specifically verifies that archives with empty-stream entries + // (files with size 0 and no compressed data) can be extracted without throwing + // NullReferenceException, with the parallel decode path opted in. + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.EmptyStream.7z"); + using var archive = SevenZipArchive.OpenArchive( + testArchive, + Parallel(ReaderOptions.ForFilePath) + ); + + var emptyStreamFileCount = 0; + foreach (var entry in archive.Entries) + { + if (!entry.IsDirectory) + { + var sevenZipEntry = entry as SevenZipEntry; + if (sevenZipEntry?.FilePart.Header.HasStream == false) + { + emptyStreamFileCount++; + } + + entry.WriteToDirectory(SCRATCH_FILES_PATH); + } + } + + Assert.True( + emptyStreamFileCount > 0, + "Test archive should contain at least one empty-stream entry" + ); + + var extractedFiles = Directory.GetFiles( + SCRATCH_FILES_PATH, + "*", + SearchOption.AllDirectories + ); + Assert.NotEmpty(extractedFiles); + + foreach (var file in extractedFiles) + { + var fileInfo = new FileInfo(file); + Assert.Equal(0, fileInfo.Length); + } + } +} diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs index 0fc86c3e..dc3b1c93 100644 --- a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs +++ b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs @@ -5,9 +5,11 @@ using SharpCompress.Archives; using SharpCompress.Archives.SevenZip; using SharpCompress.Common; using SharpCompress.Common.SevenZip; +using SharpCompress.Compressors.LZMA; using SharpCompress.Factories; using SharpCompress.Readers; using SharpCompress.Test.Mocks; +using SharpCompress.Writers.SevenZip; using Xunit; namespace SharpCompress.Test.SevenZip; @@ -445,4 +447,111 @@ public class SevenZipArchiveTests : ArchiveTests Assert.Equal(0, fileInfo.Length); } } + +#if !LEGACY_DOTNET + [Fact] + public void SevenZipArchive_EnableParallelism_OptInEngagesParallelDecodePath() + { + var archivePath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-optin.7z"); + var payload = CreateParallelDecodeTestArchive(archivePath); + + Assert.Equal( + payload, + ReadSinglePayloadEntry( + () => SevenZipArchive.OpenArchive(archivePath, ReaderOptions.ForFilePath), + out var sequentialFolderStreamType + ) + ); + Assert.NotEqual(typeof(FileStream), sequentialFolderStreamType); + + Assert.Equal( + payload, + ReadSinglePayloadEntry( + () => + SevenZipArchive.OpenArchive( + archivePath, + ReaderOptions.ForFilePath.WithEnableParallelism(true) + ), + out var parallelFolderStreamType + ) + ); + Assert.Equal(typeof(FileStream), parallelFolderStreamType); + } + + [Fact] + public void SevenZipArchive_EnableParallelism_MultipartLzma2FallsBackToSequentialDecode() + { + var archivePath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.7z"); + var payload = CreateParallelDecodeTestArchive(archivePath); + var archiveBytes = File.ReadAllBytes(archivePath); + const int splitOffset = 64; + Assert.True(archiveBytes.Length > splitOffset); + + var firstPartPath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.001"); + var secondPartPath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.002"); + File.WriteAllBytes(firstPartPath, archiveBytes[..splitOffset]); + File.WriteAllBytes(secondPartPath, archiveBytes[splitOffset..]); + var parts = new[] { new FileInfo(firstPartPath), new FileInfo(secondPartPath) }; + + Assert.Equal( + payload, + ReadSinglePayloadEntry( + () => + SevenZipArchive.OpenArchive( + parts, + ReaderOptions.ForFilePath.WithEnableParallelism(true) + ), + out var folderStreamType + ) + ); + Assert.NotEqual(typeof(FileStream), folderStreamType); + } + + private static byte[] CreateParallelDecodeTestArchive(string archivePath) + { + // Highly repetitive so it compresses well, and large enough that -- combined with the + // small dictionary below -- the encoder is forced to periodically reset the dictionary, + // producing multiple independently-decodable LZMA2 restart blocks. + var payload = new byte[20_000_000]; + for (var i = 0; i < payload.Length; i++) + { + payload[i] = (byte)(i % 7); + } + + using var archiveStream = File.Create(archivePath); + using var writer = new SevenZipWriter( + archiveStream, + new SevenZipWriterOptions(CompressionType.LZMA2) + { + LzmaProperties = new LzmaEncoderProperties( + eos: false, + dictionary: 1 << 20, + numFastBytes: 32 + ), + } + ); + using var source = new MemoryStream(payload); + writer.Write("payload.bin", source, DateTime.UtcNow); + return payload; + } + + private static byte[] ReadSinglePayloadEntry( + Func openArchive, + out Type folderStreamType + ) + { + using var archive = Assert.IsType(openArchive()); + using var reader = archive.ExtractAllEntries(); + var sevenZipReader = Assert.IsType(reader); + sevenZipReader.DiagnosticsEnabled = true; + + Assert.True(reader.MoveToNextEntry()); + using var entryStream = reader.OpenEntryStream(); + using var output = new MemoryStream(); + entryStream.CopyTo(output); + + folderStreamType = sevenZipReader.DiagnosticsCurrentFolderStream!.GetType(); + return output.ToArray(); + } +#endif } diff --git a/tests/SharpCompress.Test/Streams/BufferedSubStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/BufferedSubStreamAsyncTests.cs new file mode 100644 index 00000000..8a2125de --- /dev/null +++ b/tests/SharpCompress.Test/Streams/BufferedSubStreamAsyncTests.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class BufferedSubStreamAsyncTests +{ + [Fact] + public async ValueTask ReadAsyncByteArray_CanceledTokenWithCachedData_ThrowsOperationCanceledException() + { + using var source = new MemoryStream(new byte[] { 1, 2, 3, 4, 5 }); + using var stream = new BufferedSubStream(source, origin: 0, bytesToRead: source.Length); + var buffer = new byte[1]; + + Assert.Equal( + 1, + await stream.ReadAsync(buffer, 0, 1, CancellationToken.None).ConfigureAwait(false) + ); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => + stream.ReadAsync(buffer, 0, 1, cts.Token) + ); + } + +#if !LEGACY_DOTNET + [Fact] + public async ValueTask ReadAsyncMemory_CanceledTokenWithCachedData_ThrowsOperationCanceledException() + { + using var source = new MemoryStream(new byte[] { 1, 2, 3, 4, 5 }); + using var stream = new BufferedSubStream(source, origin: 0, bytesToRead: source.Length); + var buffer = new byte[1]; + + Assert.Equal( + 1, + await stream.ReadAsync(buffer, 0, 1, CancellationToken.None).ConfigureAwait(false) + ); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(async () => + await stream.ReadAsync(buffer.AsMemory(), cts.Token).ConfigureAwait(false) + ); + } +#endif +} diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index cc41e831..8a467f73 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -37,6 +37,47 @@ public class LzmaStreamAsyncTests : TestBase Assert.Equal((byte)'X', buffer[0]); } + [Fact] + public async Task LzmaStream_SynchronousReadThenAsynchronousRead_Throws() + { + using var stream = CreateRawLzmaStream(); + var buffer = new byte[1]; + + Assert.Equal(1, stream.Read(buffer, 0, buffer.Length)); + + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, buffer.Length) + ); + } + +#if !LEGACY_DOTNET + [Fact] + public async Task LzmaStream_AsynchronousMemoryReadThenSynchronousRead_Throws() + { + using var stream = CreateRawLzmaStream(); + var buffer = new byte[1]; + + Assert.Equal(1, await stream.ReadAsync(buffer.AsMemory())); + + Assert.Throws(() => stream.Read(buffer, 0, buffer.Length)); + } +#endif + + [Fact] + public async Task LzmaStream_ZeroLengthReadsDoNotSelectReadMode() + { + using var stream = CreateRawLzmaStream(); + var empty = Array.Empty(); + + Assert.Equal(0, stream.Read(empty, 0, 0)); + Assert.Equal(0, await stream.ReadAsync(empty, 0, 0)); + Assert.Equal(1, stream.ReadByte()); + + await Assert.ThrowsAsync(() => + stream.ReadAsync(new byte[1], 0, 1) + ); + } + private static byte[] LzmaData { get; } = [ 0x5D, @@ -625,4 +666,14 @@ public class LzmaStreamAsyncTests : TestBase } } } + + private static LzmaStream CreateRawLzmaStream() + { + var input = new MemoryStream(LzmaData); + var properties = new byte[5]; + Buffer.BlockCopy(LzmaData, 0, properties, 0, properties.Length); + var outputSize = BitConverter.ToInt64(LzmaData, 5); + input.Position = 13; + return LzmaStream.Create(properties, input, input.Length - input.Position, outputSize); + } } diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs index 8b7ebea0..887aaa94 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs @@ -1,6 +1,8 @@ using System; using System.Buffers; +using System.Collections.Generic; using System.IO; +using System.Linq; using SharpCompress.Compressors.LZMA; using Xunit; @@ -606,4 +608,471 @@ public class LzmaStreamTests } } } + + // Tests Lzma2ParallelDecoder directly using hand-built LZMA2 streams made only of + // "uncompressed" chunks (control bytes 0x01/0x02). Those chunks carry their payload verbatim + // (no real LZMA compression involved), which makes it possible to build fully valid, precisely + // controlled multi-block LZMA2 streams -- including ones that exercise the small-block merge + // threshold and cross-segment block cuts -- without needing a real (and large) 7z test archive. +#if !LEGACY_DOTNET + // Props byte encoding a ~2MB dictionary (see LzmaStream's dict-size formula), comfortably + // larger than any single chunk used by these tests. + private static readonly byte[] Lzma2Props = { 18 }; +#endif + + private static byte[] UncompressedChunk(bool dictReset, byte[] payload) + { + Assert.InRange(payload.Length, 1, 0x10000); + var sizeMinusOne = payload.Length - 1; + var chunk = new byte[3 + payload.Length]; + chunk[0] = dictReset ? (byte)0x01 : (byte)0x02; + chunk[1] = (byte)(sizeMinusOne >> 8); + chunk[2] = (byte)(sizeMinusOne & 0xFF); + Buffer.BlockCopy(payload, 0, chunk, 3, payload.Length); + return chunk; + } + + private static byte[] RepeatingPayload(int size, byte seed) + { + var payload = new byte[size]; + for (var i = 0; i < size; i++) + { + payload[i] = (byte)(seed + i); + } + return payload; + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_MergesSmallSegmentsAndCutsOnceThresholdReached() + { + // Seg1+Seg2 are both tiny -> merge. Seg3 is >= MinBlockSize on its own, so it merges into + // the same block (the cut only happens once the *already accumulated* size reaches the + // threshold), pushing that block's total past MinBlockSize. Seg4 then starts a new block, + // and Seg5 (tiny) merges into it. The trailing block is flushed as-is even though small. + var seg1 = UncompressedChunk(dictReset: true, RepeatingPayload(100, 1)); + var seg2 = UncompressedChunk(dictReset: true, RepeatingPayload(100, 2)); + var seg3 = UncompressedChunk(dictReset: true, RepeatingPayload(20_000, 3)); + var seg4 = UncompressedChunk(dictReset: true, RepeatingPayload(50, 4)); + var seg5 = UncompressedChunk(dictReset: true, RepeatingPayload(50, 5)); + + using var ms = new MemoryStream(); + ms.Write(seg1, 0, seg1.Length); + ms.Write(seg2, 0, seg2.Length); + ms.Write(seg3, 0, seg3.Length); + ms.Write(seg4, 0, seg4.Length); + ms.Write(seg5, 0, seg5.Length); + ms.WriteByte(0); // end marker + + var packSize = ms.Length; + var unpackSize = 100 + 100 + 20_000 + 50 + 50; + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, packSize, unpackSize); + + Assert.NotNull(blocks); + Assert.Equal(2, blocks!.Count); + + Assert.Equal(0, blocks[0].UnpackOffset); + Assert.Equal(100 + 100 + 20_000, blocks[0].UnpackLen); + + Assert.Equal(100 + 100 + 20_000, blocks[1].UnpackOffset); + Assert.Equal(50 + 50, blocks[1].UnpackLen); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_SingleSmallSegment_ReturnsOneBlock() + { + var seg = UncompressedChunk(dictReset: true, RepeatingPayload(64, 7)); + using var ms = new MemoryStream(); + ms.Write(seg, 0, seg.Length); + ms.WriteByte(0); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, ms.Length, 64); + + Assert.NotNull(blocks); + Assert.Single(blocks!); + Assert.Equal(0, blocks![0].UnpackOffset); + Assert.Equal(64, blocks[0].UnpackLen); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_NonRestartChunksNeverCutABlock() + { + // A dict-reset chunk followed by several *non*-restart continuation chunks must all stay + // in the same block, regardless of accumulated size, since none of them are independently + // decodable restart points. + var seg1 = UncompressedChunk(dictReset: true, RepeatingPayload(20_000, 1)); + var seg2 = UncompressedChunk(dictReset: false, RepeatingPayload(20_000, 2)); + var seg3 = UncompressedChunk(dictReset: false, RepeatingPayload(20_000, 3)); + + using var ms = new MemoryStream(); + ms.Write(seg1, 0, seg1.Length); + ms.Write(seg2, 0, seg2.Length); + ms.Write(seg3, 0, seg3.Length); + ms.WriteByte(0); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, ms.Length, 60_000); + + Assert.NotNull(blocks); + Assert.Single(blocks!); + Assert.Equal(60_000, blocks![0].UnpackLen); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_MismatchedUnpackSize_ReturnsNull() + { + var seg = UncompressedChunk(dictReset: true, RepeatingPayload(64, 7)); + using var ms = new MemoryStream(); + ms.Write(seg, 0, seg.Length); + ms.WriteByte(0); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks( + ms, + 0, + ms.Length, + 65 /* wrong */ + ); + + Assert.Null(blocks); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_InvalidControlByte_ReturnsNull() + { + using var ms = new MemoryStream(new byte[] { 0x03, 0x00, 0x00, 0x00, 0x00 }); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, ms.Length, 1); + + Assert.Null(blocks); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_TruncatedStream_ReturnsNull() + { + // Claims a 100-byte chunk but only provides a couple of bytes of payload. + using var ms = new MemoryStream(new byte[] { 0x01, 0x00, 0x63, 0xAA, 0xBB }); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, ms.Length, 100); + + Assert.Null(blocks); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_EmptyStream_ReturnsEmptyList() + { + using var ms = new MemoryStream(new byte[] { 0 }); // just the end marker + + var blocks = Lzma2ParallelDecoder.TryScanBlocks(ms, 0, ms.Length, 0); + + Assert.NotNull(blocks); + Assert.Empty(blocks!); + } + + [Fact] + public void Lzma2ParallelDecoder_TryScanBlocks_RealCompressedChunksWithNewProps_ParsesAndDecodesCorrectly() + { + // Real LZMA2 compressed chunks (as produced by the actual encoder used by SevenZipWriter) + // always set the new-props flag (control 0xE0-0xFF), which carries an extra properties + // byte in the chunk header, in addition to the 4 size-field bytes. This regression test + // guards that exact byte -- the hand-built "uncompressed chunk" tests above (control + // 0x01/0x02) never exercise this header shape, which is exactly how this bug slipped + // through: a real multi-GB archive was needed to surface it, header-only synthetic tests + // using only uncompressed chunks were not enough. + var payload = new byte[5_000_000]; + for (var i = 0; i < payload.Length; i++) + { + payload[i] = (byte)(i % 7); // highly repetitive -> compresses well, forces real chunks + } + + byte[] lzma2Props; + using var packMs = new MemoryStream(); + using ( + var encoder = new Lzma2EncoderStream(packMs, dictionarySize: 1 << 20, numFastBytes: 32) + ) + { + encoder.Write(payload, 0, payload.Length); + lzma2Props = encoder.Properties; + } + var packBytes = packMs.ToArray(); + + var blocks = Lzma2ParallelDecoder.TryScanBlocks( + packMs, + 0, + packBytes.Length, + payload.Length + ); + + Assert.NotNull(blocks); + Assert.True(blocks!.Count >= 1); + Assert.Equal(payload.Length, blocks.Sum(b => b.UnpackLen)); + + // Decode sequentially (production LzmaStream decode) to confirm the scanner's reported + // pack/unpack accounting is actually consistent with what a real decode produces. + using var decodeMs = new MemoryStream(packBytes); + using var lzma = LzmaStream.Create( + lzma2Props, + decodeMs, + -1, + payload.Length, + leaveOpen: true + ); + var decoded = new byte[payload.Length]; + var totalRead = 0; + while (totalRead < decoded.Length) + { + var read = lzma.Read(decoded, totalRead, decoded.Length - totalRead); + Assert.True(read > 0); + totalRead += read; + } + + Assert.Equal(payload, decoded); + } + +#if !LEGACY_DOTNET + [Fact] + public void Lzma2ParallelDecoder_DecodeBlocksParallel_ProducesByteIdenticalOutputAcrossMultipleBlocks() + { + // Enough independent, large-enough segments to guarantee more than one merged block. + var payloads = new List(); + for (var i = 0; i < 6; i++) + { + payloads.Add(RepeatingPayload(20_000 + i, (byte)(i * 17))); + } + + using var packMs = new MemoryStream(); + foreach (var payload in payloads) + { + var chunk = UncompressedChunk(dictReset: true, payload); + packMs.Write(chunk, 0, chunk.Length); + } + packMs.WriteByte(0); + var packBytes = packMs.ToArray(); + + var expected = new byte[payloads.Count == 0 ? 0 : payloads.Sum(p => p.Length)]; + var offset = 0; + foreach (var payload in payloads) + { + Buffer.BlockCopy(payload, 0, expected, offset, payload.Length); + offset += payload.Length; + } + + var blocks = Lzma2ParallelDecoder.TryScanBlocks( + packMs, + 0, + packBytes.Length, + expected.Length + ); + Assert.NotNull(blocks); + Assert.True(blocks!.Count > 1, "Test setup should produce more than one block."); + + var inputPath = Path.GetTempFileName(); + var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); + try + { + File.WriteAllBytes(inputPath, packBytes); + + using ( + var inputFile = new FileStream( + inputPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ) + ) + using ( + var outputFile = new FileStream( + outputPath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None + ) + ) + { + outputFile.SetLength(expected.Length); + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + Lzma2Props, + 0, + blocks, + outputFile.SafeFileHandle, + Environment.ProcessorCount, + bufferPool + ); + } + + var actual = File.ReadAllBytes(outputPath); + Assert.Equal(expected, actual); + Assert.All( + bufferPool.RentRequests, + request => Assert.InRange(request, 1, Lzma2ParallelDecoder.BlockOutputBufferSize) + ); + Assert.Equal(bufferPool.RentRequests.Count, bufferPool.ReturnedLengths.Count); + Assert.Equal(0, bufferPool.OutstandingRentals); + Assert.All(bufferPool.ClearArrayRequests, Assert.True); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + [Fact] + public void Lzma2ParallelDecoder_DecodeBlocksParallel_ReturnsPooledBuffersOnFailure() + { + var packedData = new byte[] { 0x01, 0x00, 0x3F, 0xAA, 0xBB }; + var block = new Lzma2Block(0, packedData.Length, 0, 64); + var inputPath = Path.GetTempFileName(); + var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); + + try + { + File.WriteAllBytes(inputPath, packedData); + + using ( + var inputFile = new FileStream( + inputPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ) + ) + using ( + var outputFile = new FileStream( + outputPath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None + ) + ) + { + outputFile.SetLength(block.UnpackLen); + Assert.ThrowsAny(() => + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + Lzma2Props, + 0, + [block], + outputFile.SafeFileHandle, + 1, + bufferPool + ) + ); + } + + Assert.Equal(bufferPool.RentRequests.Count, bufferPool.ReturnedLengths.Count); + Assert.Equal(0, bufferPool.OutstandingRentals); + Assert.All(bufferPool.ClearArrayRequests, Assert.True); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + [Fact] + public void Lzma2ParallelDecoder_DecodeBlocksParallel_SupportsLongPackLengths() + { + var expected = RepeatingPayload(64, 7); + var packedData = UncompressedChunk(dictReset: true, expected); + var block = new Lzma2Block(0, (long)int.MaxValue + 1, 0, expected.Length); + var inputPath = Path.GetTempFileName(); + var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); + + try + { + File.WriteAllBytes(inputPath, packedData); + + using ( + var inputFile = new FileStream( + inputPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ) + ) + using ( + var outputFile = new FileStream( + outputPath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None + ) + ) + { + outputFile.SetLength(block.UnpackLen); + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + Lzma2Props, + 0, + [block], + outputFile.SafeFileHandle, + 1, + bufferPool + ); + } + + Assert.Equal(expected, File.ReadAllBytes(outputPath)); + Assert.All( + bufferPool.RentRequests, + request => Assert.InRange(request, 1, Lzma2ParallelDecoder.BlockOutputBufferSize) + ); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + private sealed class TrackingArrayPool : ArrayPool + { + private readonly object _lock = new(); + private readonly HashSet _rented = new(); + + public List RentRequests { get; } = new(); + public List ReturnedLengths { get; } = new(); + public List ClearArrayRequests { get; } = new(); + + public int OutstandingRentals + { + get + { + lock (_lock) + { + return _rented.Count; + } + } + } + + public override byte[] Rent(int minimumLength) + { + var buffer = new byte[minimumLength]; + lock (_lock) + { + RentRequests.Add(minimumLength); + _rented.Add(buffer); + } + return buffer; + } + + public override void Return(byte[] array, bool clearArray = false) + { + lock (_lock) + { + Assert.True(_rented.Remove(array), "A buffer must be returned only once."); + ReturnedLengths.Add(array.Length); + ClearArrayRequests.Add(clearArray); + } + + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } +#endif } diff --git a/tests/SharpCompress.Test/Tar/TarArchiveAsyncTests.cs b/tests/SharpCompress.Test/Tar/TarArchiveAsyncTests.cs index 3a1b95cc..dabc0fa2 100644 --- a/tests/SharpCompress.Test/Tar/TarArchiveAsyncTests.cs +++ b/tests/SharpCompress.Test/Tar/TarArchiveAsyncTests.cs @@ -187,10 +187,6 @@ public class TarArchiveAsyncTests : ArchiveTests } } } -#if LEGACY_DOTNET - //add a delay because old .net sucks on DisposeAsync - await Task.Delay(TimeSpan.FromSeconds(1)); -#endif } [Fact] @@ -338,11 +334,8 @@ public class TarArchiveAsyncTests : ArchiveTests { ++numberOfEntries; -#if LEGACY_DOTNET - using var tarEntryStream = await entry.OpenEntryStreamAsync(); -#else - await using var tarEntryStream = await entry.OpenEntryStreamAsync(); -#endif + var tarEntryStream = await entry.OpenEntryStreamAsync(); + await using var tarEntryStreamScope = tarEntryStream.DisposeAsyncScope(); using var testFileStream = new MemoryStream(); await tarEntryStream.CopyToAsync(testFileStream); Assert.Equal(testBytes.Length, testFileStream.Length); diff --git a/tests/SharpCompress.Test/Tar/TarReaderAsyncTests.cs b/tests/SharpCompress.Test/Tar/TarReaderAsyncTests.cs index 90bce78e..75400926 100644 --- a/tests/SharpCompress.Test/Tar/TarReaderAsyncTests.cs +++ b/tests/SharpCompress.Test/Tar/TarReaderAsyncTests.cs @@ -47,22 +47,38 @@ public class TarReaderAsyncTests : ReaderTests await ReadAsync("Tar.tar.Z", CompressionType.Lzw); [Theory] - [InlineData("Tar.tar.gz", "gz", CompressionType.GZip)] - [InlineData("Tar.tar.Z", "z", CompressionType.Lzw)] + [InlineData("Tar.tar.gz", "gz", CompressionType.GZip, false)] + [InlineData("Tar.tar.gz", "gz", CompressionType.GZip, true)] + [InlineData("Tar.tar.Z", "z", CompressionType.Lzw, false)] + [InlineData("Tar.tar.Z", "z", CompressionType.Lzw, true)] public async ValueTask ReaderFactory_ExtensionHint_PreservesCompressedTarDetection_Async( string archiveName, string extensionHint, - CompressionType compressionType + CompressionType compressionType, + bool useForwardOnlyStream ) { - using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + using var file = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + Stream source = useForwardOnlyStream ? new ForwardOnlyStream(file) : file; + await using var stream = new AsyncOnlyStream(source); var options = ReaderOptions.ForExternalStream.WithExtensionHint(extensionHint); await using var reader = await ReaderFactory.OpenAsyncReader(stream, options); Assert.Equal(ArchiveType.Tar, reader.Type); - Assert.True(await reader.MoveToNextEntryAsync()); - Assert.Equal(compressionType, reader.Entry.CompressionType); + var entryCount = 0; + while (await reader.MoveToNextEntryAsync()) + { + entryCount++; + Assert.Equal(compressionType, reader.Entry.CompressionType); + if (!reader.Entry.IsDirectory) + { + await reader.WriteEntryToDirectoryAsync(SCRATCH_FILES_PATH); + } + } + + Assert.True(entryCount > 0); + VerifyFiles(); } [Fact] @@ -337,13 +353,8 @@ public class TarReaderAsyncTests : ReaderTests Assert.True(await reader.MoveToNextEntryAsync()); Assert.Equal("inner.tar.gz", reader.Entry.Key); -#if !LEGACY_DOTNET await using var entryStream = await reader.OpenEntryStreamAsync(); await using var flushingStream = new FlushOnDisposeStream(entryStream); -#else - using var entryStream = await reader.OpenEntryStreamAsync(); - using var flushingStream = new FlushOnDisposeStream(entryStream); -#endif // Extract inner.tar.gz await using var innerReader = await ReaderFactory.OpenAsyncReader(flushingStream); diff --git a/tests/SharpCompress.Test/Tar/TarWriterNonSeekableTests.cs b/tests/SharpCompress.Test/Tar/TarWriterNonSeekableTests.cs index 454c20a6..218533db 100644 --- a/tests/SharpCompress.Test/Tar/TarWriterNonSeekableTests.cs +++ b/tests/SharpCompress.Test/Tar/TarWriterNonSeekableTests.cs @@ -89,11 +89,8 @@ public class TarWriterNonSeekableTests { var entry = fileEntries.Single(e => e.Key == name); using var extracted = new MemoryStream(); -#if LEGACY_DOTNET - using (var entryStream = await entry.OpenEntryStreamAsync()) -#else - await using (var entryStream = await entry.OpenEntryStreamAsync()) -#endif + var entryStream = await entry.OpenEntryStreamAsync(); + await using (entryStream.DisposeAsyncScope()) { await entryStream.CopyToAsync(extracted); } diff --git a/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs b/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs index 5124b6b0..626265cd 100644 --- a/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs +++ b/tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs @@ -198,11 +198,7 @@ public class Zip64AsyncTests : WriterTests count++; lastKey = rd.Entry.Key; -#if LEGACY_DOTNET - using var entryStream = await rd.OpenEntryStreamAsync(); -#else await using var entryStream = await rd.OpenEntryStreamAsync(); -#endif if (rd.Entry.Key == "small") { using var ms = new MemoryStream(); @@ -337,17 +333,10 @@ public class Zip64AsyncTests : WriterTests ); while (await rd.MoveToNextEntryAsync()) { -#if LEGACY_DOTNET - using (var entryStream = await rd.OpenEntryStreamAsync()) - { - await entryStream.SkipEntryAsync(); - } -#else await using (var entryStream = await rd.OpenEntryStreamAsync()) { await entryStream.SkipEntryAsync(); } -#endif count++; if (prev != null) { diff --git a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs index cd0e610c..4a61b150 100644 --- a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs @@ -122,11 +122,8 @@ public class ZipCrcExtractionTests : ArchiveTests using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); using var archive = ZipArchive.OpenArchive(zipStream); var entry = archive.Entries.Single(e => !e.IsDirectory); -#if LEGACY_DOTNET + // MemoryStream has nothing to dispose asynchronously using var destination = new MemoryStream(); -#else - await using var destination = new MemoryStream(); -#endif var exception = await Assert.ThrowsAsync(async () => await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = true }) @@ -141,11 +138,8 @@ public class ZipCrcExtractionTests : ArchiveTests using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); using var archive = ZipArchive.OpenArchive(zipStream); var entry = archive.Entries.Single(e => !e.IsDirectory); -#if LEGACY_DOTNET + // MemoryStream has nothing to dispose asynchronously using var destination = new MemoryStream(); -#else - await using var destination = new MemoryStream(); -#endif await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = false }); diff --git a/tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs b/tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs index 4c7cd627..78f70b28 100644 --- a/tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs @@ -315,11 +315,7 @@ public class ZipReaderAsyncTests : ReaderTests { if (!reader.Entry.IsDirectory) { -#if LEGACY_DOTNET - using var entryStream = await reader.OpenEntryStreamAsync(); -#else await using var entryStream = await reader.OpenEntryStreamAsync(); -#endif // Read some data var buffer = new byte[1024]; await entryStream.ReadAsync(buffer, 0, buffer.Length); @@ -342,11 +338,7 @@ public class ZipReaderAsyncTests : ReaderTests { if (!reader.Entry.IsDirectory) { -#if LEGACY_DOTNET - using var entryStream = await reader.OpenEntryStreamAsync(); -#else await using var entryStream = await reader.OpenEntryStreamAsync(); -#endif // Read some data var buffer = new byte[1024]; await entryStream.ReadAsync(buffer, 0, buffer.Length); diff --git a/tests/SharpCompress.Test/Zip/ZipWriterNonSeekableTests.cs b/tests/SharpCompress.Test/Zip/ZipWriterNonSeekableTests.cs index 4dd10b57..95cdcc56 100644 --- a/tests/SharpCompress.Test/Zip/ZipWriterNonSeekableTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipWriterNonSeekableTests.cs @@ -163,11 +163,8 @@ public class ZipWriterNonSeekableTests { var entry = archive.Entries.Single(e => e.Key == name); using var extracted = new MemoryStream(); -#if LEGACY_DOTNET - using (var entryStream = await entry.OpenEntryStreamAsync()) -#else - await using (var entryStream = await entry.OpenEntryStreamAsync()) -#endif + var entryStream = await entry.OpenEntryStreamAsync(); + await using (entryStream.DisposeAsyncScope()) { await entryStream.CopyToAsync(extracted); } diff --git a/tests/SharpCompress.Test/packages.lock.json b/tests/SharpCompress.Test/packages.lock.json index 2be60871..43133818 100644 --- a/tests/SharpCompress.Test/packages.lock.json +++ b/tests/SharpCompress.Test/packages.lock.json @@ -4,20 +4,20 @@ ".NETFramework,Version=v4.8": { "AwesomeAssertions": { "type": "Direct", - "requested": "[9.5.0, )", - "resolved": "9.5.0", - "contentHash": "F92MsjoF8B7IdcWETru4QGZx6VAl38qQPKNGS3kwPwlLB5lb0O4bhFqG8a8sz2estQJWkKmZkVy1r19HTaFcpg==", + "requested": "[9.6.0, )", + "resolved": "9.6.0", + "contentHash": "8kdtgI+tZIgP46ncnk650/dU4JSrn+n2Dkic8FvjIFn72P6Nv1Afyr5F8F3PsokHg1GBSBVo1/Z+eZOrU5esSA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.5.4" } }, "Microsoft.NET.Test.Sdk": { "type": "Direct", - "requested": "[18.8.1, )", - "resolved": "18.8.1", - "contentHash": "dknJL3/9Y3t4XuCBqnc0PevPxgLsUMmVhjwup/b1HNovA8zWcj3XsfIf7c6p05363DWcqL7X/YhDL9B+Zymv1w==", + "requested": "[18.10.1, )", + "resolved": "18.10.1", + "contentHash": "SxCFvJE/2ltUQgCgv6+Ixagohw4sZc9hM7Wid2foaNM7paBscktoHLoRDz91fW8wbHG87Rg9Ku59/8PXumqUrA==", "dependencies": { - "Microsoft.CodeCoverage": "18.8.1" + "Microsoft.CodeCoverage": "18.10.1" } }, "Microsoft.NETFramework.ReferenceAssemblies": { @@ -31,13 +31,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -72,9 +72,9 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.ApplicationInsights": { "type": "Transitive", @@ -86,16 +86,16 @@ }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.CodeCoverage": { "type": "Transitive", - "resolved": "18.8.1", - "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" + "resolved": "18.10.1", + "contentHash": "tV7tCyp/t+DuwQugLkCeUN7NBFFB4fEGRPr0gkMQ2nQgAvQQjmGlZZi1dsLzbyoe/A0mbnKybmkIh8qhT9p4Zw==" }, "Microsoft.NETFramework.ReferenceAssemblies.net48": { "type": "Transitive", @@ -104,8 +104,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "Microsoft.Testing.Extensions.Telemetry": { "type": "Transitive", @@ -180,8 +180,8 @@ }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==", + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==", "dependencies": { "System.Buffers": "4.6.1", "System.Memory": "4.6.3" @@ -337,18 +337,18 @@ "net10.0": { "AwesomeAssertions": { "type": "Direct", - "requested": "[9.5.0, )", - "resolved": "9.5.0", - "contentHash": "F92MsjoF8B7IdcWETru4QGZx6VAl38qQPKNGS3kwPwlLB5lb0O4bhFqG8a8sz2estQJWkKmZkVy1r19HTaFcpg==" + "requested": "[9.6.0, )", + "resolved": "9.6.0", + "contentHash": "8kdtgI+tZIgP46ncnk650/dU4JSrn+n2Dkic8FvjIFn72P6Nv1Afyr5F8F3PsokHg1GBSBVo1/Z+eZOrU5esSA==" }, "Microsoft.NET.Test.Sdk": { "type": "Direct", - "requested": "[18.8.1, )", - "resolved": "18.8.1", - "contentHash": "dknJL3/9Y3t4XuCBqnc0PevPxgLsUMmVhjwup/b1HNovA8zWcj3XsfIf7c6p05363DWcqL7X/YhDL9B+Zymv1w==", + "requested": "[18.10.1, )", + "resolved": "18.10.1", + "contentHash": "SxCFvJE/2ltUQgCgv6+Ixagohw4sZc9hM7Wid2foaNM7paBscktoHLoRDz91fW8wbHG87Rg9Ku59/8PXumqUrA==", "dependencies": { - "Microsoft.CodeCoverage": "18.8.1", - "Microsoft.TestPlatform.TestHost": "18.8.1" + "Microsoft.CodeCoverage": "18.10.1", + "Microsoft.TestPlatform.TestHost": "18.10.1" } }, "Microsoft.NETFramework.ReferenceAssemblies": { @@ -362,13 +362,13 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[10.0.301, )", - "resolved": "10.0.301", - "contentHash": "S9U9Ye2bylMeDkzdNh+TsbRUNr6rkBORrnXDk45maPIxVvTmHV1SN2/qtZ/6yO7cdWmQv1LHv77yIeG6Q3nvMQ==", + "requested": "[10.0.401, )", + "resolved": "10.0.401", + "contentHash": "LGmlwgP1Cx37JEWzyjS0o1/+xs/s/e3E2TBSuogI5ePA/9L0pfIfeYX0k5in7Bfcw8Nn2y2sG9jXxydTjiR2Fg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "10.0.301", - "Microsoft.SourceLink.Common": "10.0.301", - "System.IO.Hashing": "10.0.10" + "Microsoft.Build.Tasks.Git": "10.0.401", + "Microsoft.SourceLink.Common": "10.0.401", + "System.IO.Hashing": "10.0.12" } }, "Microsoft.VisualStudio.Threading.Analyzers": { @@ -400,9 +400,9 @@ }, "Zomp.SyncMethodGenerator": { "type": "Direct", - "requested": "[2.0.42, )", - "resolved": "2.0.42", - "contentHash": "kf2PvmD9N5xIzsdf6vy5tk2w0hlD7+Kbd3w/wFmgozgHTKJlh2RNiS+Jx+4d1AtgyJVBD0CO0nofbThu5zTsgQ==" + "requested": "[2.0.56, )", + "resolved": "2.0.56", + "contentHash": "P1iMA1meu0uX8YcPWhkFnpzV3e+QFDDYMU6hMNyofS+pBlLgeVV7QevBmJmJ/mrxA1jujo3cqNaO1IapdfjOlA==" }, "Microsoft.ApplicationInsights": { "type": "Transitive", @@ -411,16 +411,16 @@ }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "9Tah8f2BYuXlff9nIsJAeQJcqnijJTXNraIcTOF2f/f9kLmg0HNWPlMemnD6FBLd/UQcE0s9Vze2zCkA6nrZAA==", + "resolved": "10.0.401", + "contentHash": "ZYctNuT10V9IYyCFydy63DXx0ggZQuynuzQOdLvW62dPgzjIz7f0ISEP75RGiq1jFQh8p6TmGSqxeQZQ87LCig==", "dependencies": { - "System.IO.Hashing": "10.0.10" + "System.IO.Hashing": "10.0.12" } }, "Microsoft.CodeCoverage": { "type": "Transitive", - "resolved": "18.8.1", - "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" + "resolved": "18.10.1", + "contentHash": "tV7tCyp/t+DuwQugLkCeUN7NBFFB4fEGRPr0gkMQ2nQgAvQQjmGlZZi1dsLzbyoe/A0mbnKybmkIh8qhT9p4Zw==" }, "Microsoft.NETFramework.ReferenceAssemblies.net461": { "type": "Transitive", @@ -429,8 +429,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "10.0.301", - "contentHash": "oz44EclJdAYVZcRb9dSbWd+6RaE+8a4j5zdE+O7yw5qPRZZhu3pcpDZ/Moy6pLeIXZqBP1FMd8CSE0sgJAGV0g==" + "resolved": "10.0.401", + "contentHash": "u3rLxIwi/9MqDFaWGE/QQgLR1NBEzLOW2lv5+9OrZPDBYIAmFdYSWCWrR1ufpXWOqFn+x02TgKropl/oDuHmgA==" }, "Microsoft.Testing.Extensions.Telemetry": { "type": "Transitive", @@ -464,15 +464,15 @@ }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", - "resolved": "18.8.1", - "contentHash": "qLbktNB1+b1XZLNJBTzaWVVJAd6PEzD7cgD406geMb6PcFZhp3EDNa1tctWx1+mtMU6MP/6ozVvFPC9vs2a9rw==" + "resolved": "18.10.1", + "contentHash": "o+PuLDRVp1iwiVAPBADZItXgm9Ck23XtPhE4kZtZVi0uYnMiiz1Dx9Cjh75kJ+ijfqnORL3VOTOaQVKi0f/ixg==" }, "Microsoft.TestPlatform.TestHost": { "type": "Transitive", - "resolved": "18.8.1", - "contentHash": "FaQHPDTUOcE+SFTjssNPfrub2lT9Zyon4J2W/KLHt/efLJACb1TCeWXyOgh0D/4Q1e4n+S3E6mOKud+9nLZlEA==", + "resolved": "18.10.1", + "contentHash": "KYz2omqkoz9E2mWa4fdgT+Ycie1wi5968EmyJlusjlp87RxxtA2FvDm8tcBEYIDzLnkWGnsKjE4d13Z7/rxkQA==", "dependencies": { - "Microsoft.TestPlatform.ObjectModel": "18.8.1" + "Microsoft.TestPlatform.ObjectModel": "18.10.1" } }, "Microsoft.Win32.Registry": { @@ -482,8 +482,8 @@ }, "System.IO.Hashing": { "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "eE839zyWZD/UBZYzP2lu9fhHX6RWe63/jz3Jxf+JumzO/db+Vc2s7CMiDwxk9ti2NTUft2tdRx31Rw2OjKbecA==" + "resolved": "10.0.12", + "contentHash": "jDix4bBMYnpZdSPcnY+KDV6ik3SRMzpMKby/bZl/XUwIiflwRNAFZ0oOl61R/pSaveIJ8t1gs2BUlrGsPs/bcg==" }, "xunit.analyzers": { "type": "Transitive", diff --git a/tests/TestArchives/Archives/Large/Large.7z b/tests/TestArchives/Archives/Large/Large.7z new file mode 100644 index 00000000..719ea18d Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.7z differ diff --git a/tests/TestArchives/Archives/Large/Large.gz b/tests/TestArchives/Archives/Large/Large.gz new file mode 100644 index 00000000..7401121e Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.gz differ diff --git a/tests/TestArchives/Archives/Large/Large.rar b/tests/TestArchives/Archives/Large/Large.rar new file mode 100644 index 00000000..6f0a65d4 Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.rar differ diff --git a/tests/TestArchives/Archives/Large/Large.tar.gz b/tests/TestArchives/Archives/Large/Large.tar.gz new file mode 100644 index 00000000..007fef08 Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.tar.gz differ diff --git a/tests/TestArchives/Archives/Large/Large.zip b/tests/TestArchives/Archives/Large/Large.zip new file mode 100644 index 00000000..5b07336f Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.zip differ diff --git a/tests/TestArchives/Archives/Large/README.md b/tests/TestArchives/Archives/Large/README.md new file mode 100644 index 00000000..0797db5d --- /dev/null +++ b/tests/TestArchives/Archives/Large/README.md @@ -0,0 +1,37 @@ +# Large Test Archives + +Each fixture contains one `large.bin` entry with a 67,108,864-byte (64 MiB) +deterministic, repeated text pattern with a 2 KiB `0xFF` prefix. The entry CRC-32 is +`f9081eb0`. + +| Fixture | Archive API | Reader API | +| --- | --- | --- | +| `Large.zip` | Yes | Yes | +| Generated `Large.tar` | Yes | Yes | +| `Large.gz` | Yes | Yes | +| `Large.rar` | Yes | Yes | +| `Large.7z` | Yes | No | +| `Large.tar.gz` | No | Yes | + +The compressible payload keeps the compressed fixtures small while requiring a full +64 MiB decompression to validate each API. The tests expand `Large.tar.gz` to a scratch +`Large.tar` before exercising raw TAR support, so the 64 MiB TAR file is not committed. + +## Regenerating + +The fixtures were created with `zip`, `tar`, `gzip`, RAR 7.22, and 7-Zip. Run these +commands from a temporary directory after replacing `` with the repository root: + +```sh +yes "SharpCompress large fixture" | head -c 67108864 > large.bin +printf '\377%.0s' {1..2048} > prefix.bin +dd if=prefix.bin of=large.bin bs=2048 count=1 conv=notrunc +touch -t 202001010000 large.bin +mkdir -p /tests/TestArchives/Archives/Large +zip -X -9 -j /tests/TestArchives/Archives/Large/Large.zip large.bin +COPYFILE_DISABLE=1 tar -cf large.tar large.bin +gzip -9 -c large.bin > /tests/TestArchives/Archives/Large/Large.gz +gzip -n -9 -c large.tar > /tests/TestArchives/Archives/Large/Large.tar.gz +rar a -ma5 -m5 -ep /tests/TestArchives/Archives/Large/Large.rar large.bin +7z a -t7z -mx=9 /tests/TestArchives/Archives/Large/Large.7z large.bin +```