diff --git a/.agents/skills/xz-lzma-format/SKILL.md b/.agents/skills/xz-lzma-format/SKILL.md new file mode 100644 index 00000000..7defd7e9 --- /dev/null +++ b/.agents/skills/xz-lzma-format/SKILL.md @@ -0,0 +1,22 @@ +--- +name: xz-lzma-format +description: Reference the XZ container format and LZMA/LZMA2 decoder behavior. Use when an AI agent needs to answer questions or make code changes involving XZ headers, blocks, indexes, checks, CRC64/XZ, LZMA2 chunks, LZMA end markers, corrupt .xz test files, or SharpCompress XZ/LZMA parsing and decompression behavior. +--- + +# XZ and LZMA Format + +Use this skill for work at the boundary between the XZ container and the LZMA/LZMA2 compression streams. It captures the key details needed for SharpCompress XZ block parsing, XZ integrity checks, and LZMA2 decoder corruption handling. + +## Reference + +- Read [references/xz-lzma-format.md](references/xz-lzma-format.md) when the task depends on XZ binary layout, XZ block checks, CRC64/XZ parameters, LZMA2 chunk control bytes, LZMA end-of-payload markers, or XZ Utils bad-file expectations. +- Treat XZ as a container around filter chains. Do not assume raw LZMA/LZMA2 behavior is equivalent to XZ stream validation. +- Use the linked XZ Utils/liblzma sources in the reference when matching corruption behavior. The test corpus contains intentionally bad files that must throw even when they can produce all expected output bytes. + +## Workflow + +1. Identify the layer involved: XZ stream header/footer, block header, compressed data padding, block check, index, filter chain, LZMA2 chunk framing, or raw LZMA range decoding. +2. Open `references/xz-lzma-format.md` and cross-check the relevant spec/source section before changing parser or decoder code. +3. For XZ checksum work, verify the stream check type from the XZ header. Use CRC32, CRC64/XZ, SHA-256, or no check according to the header, not according to a test fixture assumption. +4. For LZMA2 corruption work, compare SharpCompress behavior against the XZ Utils test corpus notes and liblzma decoder state model. +5. Test both sync and async paths. Relevant files are `XZBlock.cs`, `XZBlock.Async.cs`, `XZStream.cs`, `XZStream.Async.cs`, `LzmaStream.cs`, `LzmaStream.Async.cs`, `LzmaDecoder.cs`, and `LzmaDecoder.Async.cs`. diff --git a/.agents/skills/xz-lzma-format/references/xz-lzma-format.md b/.agents/skills/xz-lzma-format/references/xz-lzma-format.md new file mode 100644 index 00000000..60538f6f --- /dev/null +++ b/.agents/skills/xz-lzma-format/references/xz-lzma-format.md @@ -0,0 +1,174 @@ +# XZ and LZMA/LZMA2 Reference + +This reference summarizes the XZ container and LZMA/LZMA2 decoder facts that matter for SharpCompress maintenance. It is a local guide, not a full copy of the specs. + +## Upstream References + +- XZ file format specification: `https://raw.githubusercontent.com/tukaani-project/xz/master/doc/xz-file-format.txt` +- liblzma LZMA2 decoder: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/lzma/lzma2_decoder.c` +- liblzma LZMA decoder: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/lzma/lzma_decoder.c` +- XZ Utils test file descriptions: `https://raw.githubusercontent.com/tukaani-project/xz/master/tests/files/README` +- XZ range decoder reference, useful for `rc_is_finished` and normalization behavior: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/rangecoder/range_decoder.h` + +## SharpCompress Pointers + +- XZ container stream: `src/SharpCompress/Compressors/Xz/XZStream.cs`, `src/SharpCompress/Compressors/Xz/XZStream.Async.cs` +- XZ block parsing/checks: `src/SharpCompress/Compressors/Xz/XZBlock.cs`, `src/SharpCompress/Compressors/Xz/XZBlock.Async.cs` +- XZ header/footer/index: `XZHeader.cs`, `XZFooter.cs`, `XZIndex.cs`, `XZIndexRecord.cs`, and async counterparts. +- XZ LZMA2 filter wrapper: `src/SharpCompress/Compressors/Xz/Filters/Lzma2Filter.cs`, `Lzma2Filter.Async.cs` +- LZMA/LZMA2 stream and decoder: `src/SharpCompress/Compressors/LZMA/LzmaStream.cs`, `LzmaStream.Async.cs`, `LzmaDecoder.cs`, `LzmaDecoder.Async.cs`, `RangeCoder/RangeCoder.cs`, `RangeCoder/RangeCoder.Async.cs` +- Core tests: `tests/SharpCompress.Test/Xz/*`, `tests/SharpCompress.Test/Streams/LzmaStreamTests.cs`, `tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs` +- Corruption fixture discussed here: `tests/TestArchives/Archives/bad-1-lzma2-7.xz` + +## XZ Container Structure + +An XZ file is one or more XZ streams. A typical single stream is: + +```text +Stream Header -> Block(s) -> Index -> Stream Footer +``` + +Important layout rules: + +- XZ files and XZ streams are aligned to four-byte boundaries. +- Stream header magic is `FD 37 7A 58 5A 00`. +- Stream footer magic is `59 5A` (`YZ`). +- Stream header/footer flags include the check type used for every block in the stream. +- A block consists of `Block Header`, `Compressed Data`, `Block Padding`, and `Check`. +- Block padding is 0-3 null bytes and makes the block size a multiple of four. +- The Index contains one record per block: `Unpadded Size` and `Uncompressed Size`. + +## Variable-Length Integers + +XZ variable-length integers encode seven data bits per byte. The high bit means continuation. Current XZ limits the encoded integer to nine bytes/63 bits. + +SharpCompress implementation: + +- `MultiByteIntegers.ReadXZInteger` reads these values. +- It rejects overlong encodings when a continuation byte is `0x00`. +- Use this for block sizes, filter IDs, filter property sizes, index counts, and index record fields. + +## XZ Checks Versus Raw LZMA + +XZ checks are container-level integrity checks over uncompressed block data. Raw LZMA/LZMA2 decoding does not provide the same container-level CRC validation. + +Supported XZ check IDs in SharpCompress: + +- `0x00`: none, 0 bytes. +- `0x01`: CRC32, 4 bytes. +- `0x04`: CRC64/XZ, 8 bytes. +- `0x0A`: SHA-256, 32 bytes. + +CRC64/XZ details: + +- Polynomial: reflected ECMA polynomial `0xC96C5795D7870F42`. +- Initial value: `0xffffffffffffffff`. +- Final XOR: `0xffffffffffffffff`. +- Stored little-endian in the block check field. +- Test vector for `"123456789"`: `0x995DC9BBDF1939FA`. + +Common pitfall: + +- CRC64/XZ is not the older `Iso3309Polynomial = 0xD800000000000000` path that existed in SharpCompress's generic `Crc64` helper. Using that produces wrong XZ block check values. + +## XZ Block Check Handling + +When reading an `XZBlock`: + +1. Parse and CRC-validate the block header. +2. Build the filter chain in reverse order from the List of Filter Flags. +3. Read uncompressed bytes through the filter chain and update the selected check over the uncompressed bytes. +4. At block end, skip/validate block padding. +5. Read the check field and compare to the computed value. + +Important behavior: + +- A short `Stream.Read` result does not universally mean EOF. Be careful when using `bytesRead != count` as an end-of-block signal. +- Tests using `StreamReader.ReadToEnd()` often expose end-of-block behavior because they force padding/check validation. +- The check type must match the XZ stream header. For example, a fixture may have one XZ stream using CRC32 and another using CRC64; do not hard-code CRC64 in block tests. + +## LZMA2 Chunks + +LZMA2 is the only LZMA-family filter defined for XZ (`Filter ID 0x21`). Raw LZMA is not an XZ filter. + +LZMA2 control byte categories from liblzma: + +- `0x00`: LZMA2 end marker. +- `0x01` or `>= 0xE0`: dictionary reset; the next LZMA chunk must set new properties. +- `>= 0x80`: LZMA chunk. The control byte and following two bytes encode uncompressed chunk size. The next two bytes encode compressed chunk size. Some control values also provide new LZMA properties. +- `0x02`: uncompressed chunk without dictionary reset. +- `0x01`: uncompressed chunk with dictionary reset. +- `0x03..0x7F`: invalid/reserved control values. + +For LZMA chunks, the LZMA2 decoder must track: + +- Exact uncompressed chunk size. +- Exact compressed chunk size. +- Whether LZMA properties are needed. +- Whether dictionary reset is required. +- Whether the inner LZMA stream saw an LZMA end-of-payload marker. + +## LZMA End-Of-Payload Marker In LZMA2 + +The XZ Utils bad-file corpus describes `bad-1-lzma2-7.xz` as: + +```text +bad-1-lzma2-7.xz has EOPM at LZMA level. +``` + +Meaning: + +- The outer XZ container can be parsed. +- The LZMA2 stream can produce all advertised uncompressed bytes. +- The inner raw LZMA decoder still reaches an LZMA end-of-payload marker (`rep0 == uint.MaxValue`). +- LZMA2 must reject this. End-of-payload markers are for raw LZMA cases with unknown size; they are not valid as an LZMA-level terminator inside an LZMA2 chunk. + +liblzma behavior: + +- `lzma2_decoder.c` calls the inner LZMA decoder with a known `uncompressed_size` and `allow_eopm = false`. +- `lzma_decoder.c` treats EOPM as data error when EOPM is not valid. +- The XZ Utils test README says all `bad-*` files must cause decoder errors. + +SharpCompress maintenance guidance: + +- If a bad LZMA2 fixture produces all output bytes but `xz --test` reports corrupt data, inspect the inner decoder state, not just output length or XZ block check. +- In SharpCompress, `Decoder.HasEndMarker => _rep0 == uint.MaxValue` is the useful signal for the `bad-1-lzma2-7.xz` case. +- Validate both sync and async paths; `Stream.CopyToAsync` can use byte-array or `Memory` read overloads depending on target framework and wrapper stream. + +## Exception Expectations + +`DataErrorException` is internal and derives from `SharpCompressException`. Some public XZ parsing failures throw `InvalidFormatException`; raw decoder corruption may surface as `SharpCompressException` via `DataErrorException` unless the wrapper maps it. + +Testing guidance: + +- Use exact `InvalidFormatException` when the code path is XZ header/footer/block/index/check validation. +- Use `Assert.ThrowsAnyAsync` or equivalent when the desired behavior is simply that corrupt LZMA/LZMA2 data is rejected. +- Do not weaken tests to accept no exception for XZ Utils `bad-*` fixtures. + +## Useful Commands + +Use the system `xz` tool as an oracle when available: + +```bash +xz --test --verbose tests/TestArchives/Archives/bad-1-lzma2-7.xz +xz --robot --list --verbose tests/TestArchives/Archives/bad-1-lzma2-7.xz +``` + +Expected for `bad-1-lzma2-7.xz`: + +```text +xz: tests/TestArchives/Archives/bad-1-lzma2-7.xz: Compressed data is corrupt +``` + +Targeted SharpCompress tests: + +```bash +dotnet test tests/SharpCompress.Test/SharpCompress.Test.csproj --framework net10.0 --filter "FullyQualifiedName~SharpCompress.Test.Streams.LzmaStream|FullyQualifiedName~SharpCompress.Test.Xz" +``` + +## Current SharpCompress Gotchas + +- XZ index CRC32 verification may still be incomplete; check `XZIndex.VerifyCrc32` before relying on index corruption detection. +- XZ block/index size semantics are easy to confuse. `Unpadded Size` excludes block padding but includes header, compressed data, and check. `Uncompressed Size` is raw output size. +- LZMA2 chunks include their own compressed and uncompressed chunk sizes; these are separate from XZ block header/index sizes. +- `StreamReader.ReadToEnd()` and `TransferToAsync(Stream.Null, long.MaxValue)` are useful for forcing full-stream validation. diff --git a/docs/API.md b/docs/API.md index 19edd6ed..db807f55 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,6 +132,15 @@ using (var archive = ZipArchive.OpenArchive("file.zip")) var firstEntry = archive.Entries.First(); firstEntry.WriteToFile(@"C:\output\file.txt"); + // Extract single entry to a stream with extraction options + using (var outputStream = File.Create(@"C:\output\file.txt")) + { + firstEntry.WriteTo( + outputStream, + new ExtractionOptions { CheckCrc = false } + ); + } + // Get entry stream using (var stream = entry.OpenEntryStream()) { @@ -154,6 +163,15 @@ await using (var asyncArchive = await ZipArchive.OpenAsyncArchive("file.zip")) { await foreach (var entry in asyncArchive.EntriesAsync) { + await using (var outputStream = File.Create(@"C:\output\" + entry.Key)) + { + await entry.WriteToAsync( + outputStream, + new ExtractionOptions { CheckCrc = false }, + cancellationToken: cancellationToken + ); + } + using (var stream = await entry.OpenEntryStreamAsync(cancellationToken)) { // ... @@ -320,7 +338,11 @@ var flatOptions = ExtractionOptions.FlatExtract; // No directory structure var metadataOptions = ExtractionOptions.PreserveMetadata; // Keep timestamps and attributes // Tune extraction copy buffering -var extractionOptions = new ExtractionOptions { BufferSize = 131072 }; +var extractionOptions = new ExtractionOptions +{ + BufferSize = 131072, + CheckCrc = true, // Default: validate entry checksums when archive metadata provides one +}; // Factory defaults: // - file path / FileInfo overloads use LeaveStreamOpen = false @@ -429,7 +451,8 @@ var options = new ExtractionOptions { ExtractFullPath = true, // Recreate directory structure Overwrite = true, // Overwrite existing files - PreserveFileTime = true // Keep original timestamps + PreserveFileTime = true, // Keep original timestamps + CheckCrc = true // Validate payload checksums when available }; using (var archive = ZipArchive.OpenArchive("file.zip")) @@ -438,6 +461,8 @@ using (var archive = ZipArchive.OpenArchive("file.zip")) } ``` +`CheckCrc` validates archive-level payload checksums when the format stores reliable metadata, such as ZIP CRC32 values. Formats without payload checksums skip this validation. Decompressor integrity checks that are required to decode a stream may still fail even when `CheckCrc` is disabled. + ### Options matrix ```text diff --git a/docs/FORMATS.md b/docs/FORMATS.md index 5b149b48..3504d659 100644 --- a/docs/FORMATS.md +++ b/docs/FORMATS.md @@ -48,9 +48,15 @@ 2. Skip through the decompressed data to reach each file's starting position **Recommendation**: For best performance with 7Zip archives, use synchronous extraction methods (`MoveToNextEntry()` and `WriteEntryToDirectory()`) when possible. Use async methods only when you need to avoid blocking the thread (e.g., in UI applications or async-only contexts). - + **Technical Details**: 7Zip archives group files into "folders" (compression units), where all files in a folder share one continuous LZMA-compressed stream. The LZMA decoder maintains internal state (dictionary window, decoder positions) that assumes sequential, non-interruptible processing. Async operations can yield control during awaits, which would corrupt this shared state. To avoid this, async extraction creates a fresh decoder stream for each file. +### XZ Format Notes + +- XZ is a container format around LZMA2-compressed blocks, not just raw LZMA/LZMA2 data. +- XZ streams can include per-block integrity checks selected by the stream header: CRC32, CRC64/XZ, SHA-256, or none. SharpCompress validates these checks while reading XZ blocks. +- Raw LZMA/LZMA2 decoding does not provide the same container-level CRC validation; it only validates what the decoder format itself can detect, such as malformed compressed data or invalid end markers. + ## Compression Streams For those who want to directly compress/decompress bits. The single file formats are represented here as well. However, BZip2, LZip and XZ have no metadata (GZip has a little) so using them without something like a Tar file makes little sense. diff --git a/docs/USAGE.md b/docs/USAGE.md index 72356614..9666d43f 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -105,6 +105,7 @@ using (var archive = RarArchive.OpenArchive("Test.rar", ReaderOptions.ForFilePat ExtractFullPath = true, Overwrite = true, BufferSize = 131072, + CheckCrc = true, // Default: validate payload checksums when available } ); } @@ -141,7 +142,7 @@ using (var archive = RarArchive.OpenArchive("archive.rar", { archive.WriteToDirectory( @"D:\output", - new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + new ExtractionOptions { ExtractFullPath = true, Overwrite = true, CheckCrc = true } ); } ``` diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 0de4b21c..502a419e 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -18,11 +18,24 @@ public static class IArchiveEntryExtensions /// The stream to write the entry content to. /// Optional progress reporter for tracking extraction progress. public void WriteTo(Stream streamToWriteTo, IProgress? progress = null) => - archiveEntry.WriteTo(streamToWriteTo, null, progress); + archiveEntry.WriteTo(streamToWriteTo, bufferSize: null, progress: progress); + + /// + /// Extract entry to the specified stream. + /// + /// The stream to write the entry content to. + /// Options for configuring extraction behavior. + /// Optional progress reporter for tracking extraction progress. + public void WriteTo( + Stream streamToWriteTo, + ExtractionOptions options, + IProgress? progress = null + ) => archiveEntry.WriteTo(streamToWriteTo, options.BufferSize, options, progress); private void WriteTo( Stream streamToWriteTo, int? bufferSize, + ExtractionOptions? options = null, IProgress? progress = null ) { @@ -32,7 +45,10 @@ public static class IArchiveEntryExtensions } using var entryStream = archiveEntry.OpenEntryStream(); - var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); + var checkedStream = options is null + ? entryStream + : IEntryExtensions.WrapWithChecksumValidation(archiveEntry, entryStream, options); + var sourceStream = WrapWithProgress(checkedStream, archiveEntry, progress); sourceStream.CopyTo(streamToWriteTo, bufferSize ?? Constants.BufferSize); } @@ -46,16 +62,43 @@ public static class IArchiveEntryExtensions Stream streamToWriteTo, IProgress? progress = null, CancellationToken cancellationToken = default - ) - { + ) => await archiveEntry - .WriteToAsync(streamToWriteTo, Constants.BufferSize, progress, cancellationToken) + .WriteToAsync( + streamToWriteTo, + Constants.BufferSize, + progress: progress, + cancellationToken: cancellationToken + ) + .ConfigureAwait(false); + + /// + /// Extract entry to the specified stream asynchronously. + /// + /// The stream to write the entry content to. + /// Options for configuring extraction behavior. + /// Optional progress reporter for tracking extraction progress. + /// Cancellation token. + public async ValueTask WriteToAsync( + Stream streamToWriteTo, + ExtractionOptions options, + IProgress? progress = null, + CancellationToken cancellationToken = default + ) => + await archiveEntry + .WriteToAsync( + streamToWriteTo, + options.BufferSize, + options, + progress, + cancellationToken + ) .ConfigureAwait(false); - } private async ValueTask WriteToAsync( Stream streamToWriteTo, int? bufferSize, + ExtractionOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default ) @@ -74,7 +117,10 @@ public static class IArchiveEntryExtensions .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); #endif - var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); + var checkedStream = options is null + ? entryStream + : IEntryExtensions.WrapWithChecksumValidation(archiveEntry, entryStream, options); + var sourceStream = WrapWithProgress(checkedStream, archiveEntry, progress); await sourceStream .CopyToAsync(streamToWriteTo, bufferSize ?? Constants.BufferSize, cancellationToken) .ConfigureAwait(false); @@ -154,13 +200,14 @@ public static class IArchiveEntryExtensions /// public void WriteToFile(string destinationFileName, ExtractionOptions? options = null) { + options ??= new ExtractionOptions(); entry.WriteEntryToFile( destinationFileName, options, (x, fm) => { using var fs = File.Open(x, fm); - entry.WriteTo(fs, options?.BufferSize ?? Constants.BufferSize); + entry.WriteTo(fs, options?.BufferSize ?? Constants.BufferSize, options, null); } ); } @@ -172,7 +219,9 @@ public static class IArchiveEntryExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await entry .WriteEntryToFileAsync( destinationFileName, @@ -181,11 +230,12 @@ public static class IArchiveEntryExtensions { using var fs = File.Open(x, fm); await entry - .WriteToAsync(fs, options?.BufferSize, null, ct) + .WriteToAsync(fs, options.BufferSize, options, null, ct) .ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } } } diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index d147264e..021381c3 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -71,22 +71,19 @@ public partial class RarArchiveEntry : RarEntry, IArchiveEntry public Stream OpenEntryStream() { + var readStream = new MultiVolumeReadOnlyStream(Parts.Cast()); RarStream stream; if (IsRarV3) { - stream = new RarStream( - archive.UnpackV1.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast()) - ); + stream = RarCrcStream.Create(archive.UnpackV1.Value, FileHeader, readStream); + } + else if (FileHeader.FileCrc?.Length > 5) + { + stream = RarBLAKE2spStream.Create(archive.UnpackV2017.Value, FileHeader, readStream); } else { - stream = new RarStream( - archive.UnpackV2017.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast()) - ); + stream = RarCrcStream.Create(archive.UnpackV2017.Value, FileHeader, readStream); } stream.Initialize(); diff --git a/src/SharpCompress/Common/Ace/AceEntry.cs b/src/SharpCompress/Common/Ace/AceEntry.cs index 419329de..0164d229 100644 --- a/src/SharpCompress/Common/Ace/AceEntry.cs +++ b/src/SharpCompress/Common/Ace/AceEntry.cs @@ -31,6 +31,14 @@ public class AceEntry : Entry } } + internal override ChecksumDescriptor Checksum => + !IsDirectory + && !IsEncrypted + && !_filePart.Header.IsContinuedFromPrev + && !_filePart.Header.IsContinuedToNext + ? new ChecksumDescriptor(ChecksumKind.Crc32NoFinalXor, _filePart.Header.Crc32, true) + : default; + public override string? Key => _filePart?.Header.Filename; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arc/ArcEntry.cs b/src/SharpCompress/Common/Arc/ArcEntry.cs index cb7b262c..9627606d 100644 --- a/src/SharpCompress/Common/Arc/ArcEntry.cs +++ b/src/SharpCompress/Common/Arc/ArcEntry.cs @@ -32,6 +32,11 @@ public class ArcEntry : Entry } } + internal override ChecksumDescriptor Checksum => + _filePart is not null && _filePart.Header.CompressionMethod != CompressionType.Unknown + ? new ChecksumDescriptor(ChecksumKind.Crc16Arc, _filePart.Header.Crc16, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arj/ArjEntry.cs b/src/SharpCompress/Common/Arj/ArjEntry.cs index cf5e1c9f..f0159a66 100644 --- a/src/SharpCompress/Common/Arj/ArjEntry.cs +++ b/src/SharpCompress/Common/Arj/ArjEntry.cs @@ -21,6 +21,13 @@ public class ArjEntry : Entry public override long Crc => _filePart.Header.OriginalCrc32; + internal override ChecksumDescriptor Checksum => + !IsDirectory + && _filePart.Header.CompressionMethod != CompressionMethod.NoDataNoCrc + && _filePart.Header.CompressionMethod != CompressionMethod.NoData + ? new ChecksumDescriptor(ChecksumKind.Crc32, _filePart.Header.OriginalCrc32, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/ChecksumDescriptor.cs b/src/SharpCompress/Common/ChecksumDescriptor.cs new file mode 100644 index 00000000..b418df2b --- /dev/null +++ b/src/SharpCompress/Common/ChecksumDescriptor.cs @@ -0,0 +1,14 @@ +namespace SharpCompress.Common; + +internal enum ChecksumKind +{ + Crc32, + Crc32NoFinalXor, + Crc16Arc, +} + +internal readonly record struct ChecksumDescriptor( + ChecksumKind Kind, + long ExpectedValue, + bool IsAvailable +); diff --git a/src/SharpCompress/Common/ChecksumValidationStream.cs b/src/SharpCompress/Common/ChecksumValidationStream.cs new file mode 100644 index 00000000..1b14cc9d --- /dev/null +++ b/src/SharpCompress/Common/ChecksumValidationStream.cs @@ -0,0 +1,178 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Crypto; + +namespace SharpCompress.Common; + +internal sealed class ChecksumValidationStream : Stream +{ + private readonly Stream _stream; + private readonly ChecksumDescriptor _checksum; + private readonly string _entryName; + private readonly uint[] _crc32Table; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private ushort _crc16; + private bool _validated; + + internal ChecksumValidationStream(Stream stream, ChecksumDescriptor checksum, string? entryName) + { + _stream = stream; + _checksum = checksum; + _entryName = string.IsNullOrEmpty(entryName) ? "Entry" : entryName!; + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _stream.Length; + + public override long Position + { + get => _stream.Position; + set => throw new NotSupportedException(); + } + + public override void Flush() => _stream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _stream.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } +#endif + + public override int ReadByte() => throw new NotSupportedException(); + + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (read > 0) + { + UpdateChecksum(buffer); + return; + } + + Validate(); + } + + private void UpdateChecksum(ReadOnlySpan buffer) + { + switch (_checksum.Kind) + { + case ChecksumKind.Crc32: + case ChecksumKind.Crc32NoFinalXor: + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + break; + case ChecksumKind.Crc16Arc: + _crc16 = CalculateCrc16Arc(_crc16, buffer); + break; + } + } + + private void Validate() + { + if (_validated) + { + return; + } + + _validated = true; + + switch (_checksum.Kind) + { + case ChecksumKind.Crc32: + ValidateCrc32(finalXor: true); + break; + case ChecksumKind.Crc32NoFinalXor: + ValidateCrc32(finalXor: false); + break; + case ChecksumKind.Crc16Arc: + ValidateCrc16Arc(); + break; + } + } + + private void ValidateCrc32(bool finalXor) + { + var actual = finalXor ? ~_seed : _seed; + var expected = unchecked((uint)_checksum.ExpectedValue); + if (actual != expected) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expected:X8}, actual 0x{actual:X8}." + ); + } + } + + private void ValidateCrc16Arc() + { + var expected = unchecked((ushort)_checksum.ExpectedValue); + if (_crc16 != expected) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expected:X4}, actual 0x{_crc16:X4}." + ); + } + } + + private static ushort CalculateCrc16Arc(ushort crc, ReadOnlySpan buffer) + { + foreach (var value in buffer) + { + crc ^= value; + for (var i = 0; i < 8; i++) + { + crc = (crc & 1) != 0 ? (ushort)((crc >> 1) ^ 0xA001) : (ushort)(crc >> 1); + } + } + + return crc; + } +} diff --git a/src/SharpCompress/Common/Entry.cs b/src/SharpCompress/Common/Entry.cs index 1942ba46..43095b71 100644 --- a/src/SharpCompress/Common/Entry.cs +++ b/src/SharpCompress/Common/Entry.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using SharpCompress.Common.Options; @@ -84,6 +85,19 @@ public abstract class Entry : IEntry internal virtual void Close() { } + internal virtual ChecksumDescriptor Checksum => default; + + internal virtual Stream WrapWithChecksumValidation(Stream source, ExtractionOptions options) + { + var checksum = Checksum; + if (!checksum.IsAvailable) + { + return source; + } + + return new ChecksumValidationStream(source, checksum, Key); + } + /// /// Entry file attribute. /// diff --git a/src/SharpCompress/Common/ExtractionOptions.cs b/src/SharpCompress/Common/ExtractionOptions.cs index b99658f8..04b33416 100644 --- a/src/SharpCompress/Common/ExtractionOptions.cs +++ b/src/SharpCompress/Common/ExtractionOptions.cs @@ -43,6 +43,15 @@ public sealed record ExtractionOptions : IExtractionOptions /// public int BufferSize { get; set; } = Constants.BufferSize; + /// + /// Validate archive entry checksums during extraction when checksum metadata is available. + /// + /// + /// Formats without payload checksums skip this validation. Compression-format integrity + /// checks that are required to decode data may still fail even when this is disabled. + /// + public bool CheckCrc { get; set; } = true; + /// /// Delegate for writing symbolic links to disk. /// The first parameter is the source path (where the symlink is created). diff --git a/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs new file mode 100644 index 00000000..512d7cc2 --- /dev/null +++ b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs @@ -0,0 +1,165 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Crypto; + +namespace SharpCompress.Common.GZip; + +internal sealed class GZipChecksumValidationStream : Stream +{ + private readonly Stream _source; + private readonly Stream _rawStream; + private readonly string _entryName; + private readonly uint? _expectedCrc; + private readonly uint? _expectedSize; + private readonly uint[] _crc32Table; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private uint _size; + private bool _validated; + + internal GZipChecksumValidationStream( + Stream source, + Stream rawStream, + string? entryName, + uint? expectedCrc, + uint? expectedSize + ) + { + _source = source; + _rawStream = rawStream; + _entryName = string.IsNullOrEmpty(entryName) ? "Entry" : entryName!; + _expectedCrc = expectedCrc; + _expectedSize = expectedSize; + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + } + + public override bool CanRead => _source.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _source.Length; + + public override long Position + { + get => _source.Position; + set => throw new NotSupportedException(); + } + + public override void Flush() => _source.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _source.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _source.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _source.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } +#endif + + public override int ReadByte() + { + var value = _source.ReadByte(); + if (value == -1) + { + Validate(); + } + else + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, (byte)value); + _size++; + } + + return value; + } + + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _source + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (read > 0) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + _size += unchecked((uint)read); + return; + } + + Validate(); + } + + private void Validate() + { + if (_validated) + { + return; + } + + _validated = true; + + var expectedCrc = _expectedCrc; + var expectedSize = _expectedSize; + if (!expectedCrc.HasValue || !expectedSize.HasValue) + { + Span trailer = stackalloc byte[8]; + _rawStream.ReadFully(trailer); + expectedCrc = BinaryPrimitives.ReadUInt32LittleEndian(trailer); + expectedSize = BinaryPrimitives.ReadUInt32LittleEndian(trailer[4..]); + } + + var actualCrc = ~_seed; + if (actualCrc != expectedCrc.Value) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expectedCrc.Value:X8}, actual 0x{actualCrc:X8}." + ); + } + + if (_size != expectedSize.Value) + { + throw new InvalidFormatException( + $"Size mismatch for entry '{_entryName}'. Expected {expectedSize.Value}, actual {_size}." + ); + } + } +} diff --git a/src/SharpCompress/Common/GZip/GZipEntry.cs b/src/SharpCompress/Common/GZip/GZipEntry.cs index 635fc4cd..48f08d46 100644 --- a/src/SharpCompress/Common/GZip/GZipEntry.cs +++ b/src/SharpCompress/Common/GZip/GZipEntry.cs @@ -20,6 +20,9 @@ public partial class GZipEntry : Entry public override long Crc => _filePart?.Crc ?? 0; + internal override Stream WrapWithChecksumValidation(Stream source, ExtractionOptions options) => + _filePart?.WrapWithChecksumValidation(source, Key) ?? source; + public override string? Key => _filePart?.FilePartName; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/GZip/GZipFilePart.cs b/src/SharpCompress/Common/GZip/GZipFilePart.cs index e8d89a5d..13a1c968 100644 --- a/src/SharpCompress/Common/GZip/GZipFilePart.cs +++ b/src/SharpCompress/Common/GZip/GZipFilePart.cs @@ -5,6 +5,7 @@ using System.IO; using SharpCompress.Common.Tar.Headers; using SharpCompress.Compressors; using SharpCompress.Compressors.Deflate; +using SharpCompress.IO; using SharpCompress.Providers; namespace SharpCompress.Common.GZip; @@ -48,7 +49,7 @@ internal sealed partial class GZipFilePart : FilePart ) : base(archiveEncoding) { - _stream = stream; + _stream = SharpCompressStream.Create(stream); _compressionProviders = compressionProviders; } @@ -66,6 +67,9 @@ internal sealed partial class GZipFilePart : FilePart return _compressionProviders.CreateDecompressStream(CompressionType.Deflate, _stream); } + internal Stream WrapWithChecksumValidation(Stream source, string? entryName) => + new GZipChecksumValidationStream(source, _stream, entryName, Crc, UncompressedSize); + internal override Stream GetRawStream() => _stream; private void ReadTrailer() diff --git a/src/SharpCompress/Common/IEntryExtensions.cs b/src/SharpCompress/Common/IEntryExtensions.cs index d3158f03..492ad85d 100644 --- a/src/SharpCompress/Common/IEntryExtensions.cs +++ b/src/SharpCompress/Common/IEntryExtensions.cs @@ -5,6 +5,21 @@ namespace SharpCompress.Common; internal static partial class IEntryExtensions { + internal static Stream WrapWithChecksumValidation( + IEntry entry, + Stream source, + ExtractionOptions? options + ) + { + options ??= new ExtractionOptions(); + if (options.CheckCrc && entry is Entry typedEntry) + { + return typedEntry.WrapWithChecksumValidation(source, options); + } + + return source; + } + extension(IEntry entry) { /// diff --git a/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs b/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs index fb6d23fa..576c1d16 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs @@ -18,6 +18,28 @@ public class SevenZipEntry : Entry public override long Crc => FilePart.Header.Crc ?? 0; + internal override ChecksumDescriptor Checksum + { + get + { + if ( + IsDirectory + || FilePart.Header.IsAnti + || !FilePart.Header.HasStream + || !FilePart.Header.Crc.HasValue + ) + { + return default; + } + + return new ChecksumDescriptor( + ChecksumKind.Crc32, + FilePart.Header.Crc.Value, + IsAvailable: true + ); + } + } + public override string? Key => FilePart.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs index c09cac91..af978f64 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs @@ -22,6 +22,7 @@ internal partial class DirectoryEntryHeader .ReadUInt16Async() .ConfigureAwait(false); Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + IsCrcAvailable = true; CompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); UncompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); var nameLength = await reader.ReadUInt16Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs index e4e0f331..f41c6047 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs @@ -17,6 +17,7 @@ internal partial class DirectoryEntryHeader : ZipFileEntry OriginalLastModifiedTime = LastModifiedTime = reader.ReadUInt16(); OriginalLastModifiedDate = LastModifiedDate = reader.ReadUInt16(); Crc = reader.ReadUInt32(); + IsCrcAvailable = true; CompressedSize = reader.ReadUInt32(); UncompressedSize = reader.ReadUInt32(); var nameLength = reader.ReadUInt16(); diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs index 950494df..2e96c942 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs @@ -20,6 +20,7 @@ internal partial class LocalEntryHeader .ReadUInt16Async() .ConfigureAwait(false); Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + IsCrcAvailable = !Flags.HasFlag(HeaderFlags.UsePostDataDescriptor); CompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); UncompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); var nameLength = await reader.ReadUInt16Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs index d9490137..9d4512b1 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs @@ -16,6 +16,7 @@ internal partial class LocalEntryHeader : ZipFileEntry OriginalLastModifiedTime = LastModifiedTime = reader.ReadUInt16(); OriginalLastModifiedDate = LastModifiedDate = reader.ReadUInt16(); Crc = reader.ReadUInt32(); + IsCrcAvailable = !Flags.HasFlag(HeaderFlags.UsePostDataDescriptor); CompressedSize = reader.ReadUInt32(); UncompressedSize = reader.ReadUInt32(); var nameLength = reader.ReadUInt16(); diff --git a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs index b62b6a67..e7f0b042 100644 --- a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs +++ b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs @@ -80,6 +80,8 @@ internal abstract partial class ZipFileEntry(ZipHeaderType type, IArchiveEncodin internal uint Crc { get; set; } + internal bool IsCrcAvailable { get; set; } + protected void LoadExtra(byte[] extra) { for (var i = 0; i < extra.Length; ) diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs index 22304b9d..7a88a480 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs @@ -159,6 +159,7 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory if (FlagUtility.HasFlag(localEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor)) { localEntryHeader.Crc = directoryEntryHeader.Crc; + localEntryHeader.IsCrcAvailable = true; localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; } diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs index 80ff16b4..4e3bcc3a 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs @@ -124,6 +124,7 @@ internal sealed partial class StreamingZipHeaderFactory .ConfigureAwait(false); } lastEntryHeader.Crc = crc; + lastEntryHeader.IsCrcAvailable = true; //attempt 32bit read ulong compressedSize = await _reader @@ -205,6 +206,7 @@ internal sealed partial class StreamingZipHeaderFactory .ConfigureAwait(false); } lastEntryHeader.Crc = crc; + lastEntryHeader.IsCrcAvailable = true; // The DataDescriptor can be either 64bit or 32bit var compressedSize = await _reader @@ -283,6 +285,7 @@ internal sealed partial class StreamingZipHeaderFactory localHeader.UncompressedSize = directoryHeader.Size; localHeader.CompressedSize = directoryHeader.CompressedSize; localHeader.Crc = (uint)directoryHeader.Crc; + localHeader.IsCrcAvailable = true; } // If we have CompressedSize, there is data to be read diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs index 7dcc4c54..c955afb0 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs @@ -59,6 +59,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory crc = reader.ReadUInt32(); } _lastEntryHeader.Crc = crc; + _lastEntryHeader.IsCrcAvailable = true; //attempt 32bit read ulong compSize = reader.ReadUInt32(); @@ -121,6 +122,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory crc = reader.ReadUInt32(); } _lastEntryHeader.Crc = crc; + _lastEntryHeader.IsCrcAvailable = true; // The DataDescriptor can be either 64bit or 32bit var compressed_size = reader.ReadUInt32(); @@ -188,6 +190,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory local_header.UncompressedSize = dir_header.Size; local_header.CompressedSize = dir_header.CompressedSize; local_header.Crc = (uint)dir_header.Crc; + local_header.IsCrcAvailable = true; } // If we have CompressedSize, there is data to be read diff --git a/src/SharpCompress/Common/Zip/ZipEntry.cs b/src/SharpCompress/Common/Zip/ZipEntry.cs index 753c6dc7..bfe1a50d 100644 --- a/src/SharpCompress/Common/Zip/ZipEntry.cs +++ b/src/SharpCompress/Common/Zip/ZipEntry.cs @@ -89,6 +89,49 @@ public class ZipEntry : Entry public override long Crc => _filePart?.Header.Crc ?? 0; + internal override ChecksumDescriptor Checksum + { + get + { + if ( + _filePart is null + || IsDirectory + || !_filePart.Header.IsCrcAvailable + || !IsReliableCrcMetadata(_filePart.Header) + ) + { + return default; + } + + return new ChecksumDescriptor( + ChecksumKind.Crc32, + _filePart.Header.Crc, + IsAvailable: true + ); + } + } + + private static bool IsReliableCrcMetadata(ZipFileEntry header) + { + if (header.CompressionMethod != ZipCompressionMethod.WinzipAes) + { + return true; + } + + var aesExtraData = header.Extra.FirstOrDefault(x => x.Type == ExtraDataType.WinZipAes); + if (aesExtraData is null || aesExtraData.DataBytes.Length < MinimumWinZipAesExtraDataLength) + { + return false; + } + + var vendorVersion = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian( + aesExtraData.DataBytes + ); + + // WinZip AES AE-2 stores a zero CRC field by design and relies on AES authentication. + return vendorVersion == 0x0001; + } + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs index c31e24a6..bc5944b4 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs @@ -44,6 +44,7 @@ internal partial class ZipHeaderFactory ) { _lastEntryHeader.Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + _lastEntryHeader.IsCrcAvailable = true; _lastEntryHeader.CompressedSize = zip64 ? (long)await reader.ReadUInt64Async().ConfigureAwait(false) : await reader.ReadUInt32Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs index 8d0ffdb4..e1ebb8c1 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs @@ -66,6 +66,7 @@ internal partial class ZipHeaderFactory ) { _lastEntryHeader.Crc = reader.ReadUInt32(); + _lastEntryHeader.IsCrcAvailable = true; _lastEntryHeader.CompressedSize = zip64 ? (long)reader.ReadUInt64() : reader.ReadUInt32(); diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs index 2cc7fb8d..949fbc97 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs @@ -169,7 +169,7 @@ public sealed partial class LZipStream public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, cancellationToken); + ) => ReadAndValidateAsync(buffer, cancellationToken); #endif /// @@ -180,7 +180,33 @@ public sealed partial class LZipStream int offset, int count, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, offset, count, cancellationToken); + ) => ReadAndValidateAsync(buffer, offset, count, cancellationToken); + + private async Task ReadAndValidateAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + private async ValueTask ReadAndValidateAsync( + Memory buffer, + CancellationToken cancellationToken + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif /// /// Asynchronously writes bytes from a buffer to the current stream. diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.cs index 618f3b57..f2e1abea 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.cs @@ -22,8 +22,18 @@ public sealed partial class LZipStream : Stream, IFinishable { private readonly Stream _stream; private readonly CountingStream? _countingWritableSubStream; + private readonly CountingStream? _countingReadableSubStream; + private readonly uint[]? _crc32Table; + private readonly ulong? _expectedDataSize; + private readonly ulong? _expectedMemberSize; + private readonly bool _skipTrailerValidation; private bool _disposed; private bool _finished; + private bool _trailerValidated; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private ulong _readCount; + private readonly long _memberStartPosition; + private readonly long _compressedDataStartPosition; private long _writeCount; private readonly Stream? _originalStream; @@ -37,13 +47,43 @@ public sealed partial class LZipStream : Stream, IFinishable if (mode == CompressionMode.Decompress) { + _skipTrailerValidation = stream is SharpCompressStream; + _memberStartPosition = stream.CanSeek ? stream.Position : 0; var dSize = ValidateAndReadSize(stream); if (dSize == 0) { throw new InvalidFormatException("Not an LZip stream"); } var properties = GetProperties(dSize); - _stream = LzmaStream.Create(properties, stream, leaveOpen: leaveOpen); + var trailerStream = GetSeekableTrailerStream(stream); + if (trailerStream is not null) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 16; + Span sizeTrailer = stackalloc byte[16]; + trailerStream.ReadFully(sizeTrailer); + _expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer); + _expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer[8..]); + if (_expectedDataSize > long.MaxValue) + { + throw new InvalidFormatException("LZip data size is too large."); + } + trailerStream.Position = position; + } + _compressedDataStartPosition = stream.CanSeek ? stream.Position : 0; + _countingReadableSubStream = new CountingStream( + SharpCompressStream.CreateNonDisposing(stream) + ); + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + _stream = LzmaStream.Create( + properties, + _countingReadableSubStream, + inputSize: -1, + outputSize: _expectedDataSize.HasValue + ? checked((long)_expectedDataSize.Value) + : -1, + leaveOpen: leaveOpen + ); } else { @@ -106,7 +146,7 @@ public sealed partial class LZipStream : Stream, IFinishable { Finish(); _stream.Dispose(); - if (Mode == CompressionMode.Compress && !_leaveOpen) + if (!_leaveOpen) { _originalStream?.Dispose(); } @@ -134,10 +174,29 @@ public sealed partial class LZipStream : Stream, IFinishable set => throw new NotImplementedException(); } - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } - public override int ReadByte() => _stream.ReadByte(); + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value == -1) + { + ValidateTrailer(); + } + else + { + Span buffer = stackalloc byte[1]; + buffer[0] = (byte)value; + UpdateChecksum(buffer); + } + + return value; + } public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); @@ -145,7 +204,12 @@ public sealed partial class LZipStream : Stream, IFinishable #if !LEGACY_DOTNET - public override int Read(Span buffer) => _stream.Read(buffer); + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } public override void Write(ReadOnlySpan buffer) { @@ -246,4 +310,130 @@ public sealed partial class LZipStream : Stream, IFinishable (byte)((dictionarySize >> 16) & 0xff), (byte)((dictionarySize >> 24) & 0xff), ]; + + private static Stream? GetSeekableTrailerStream(Stream stream) + { + while (stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream) + { + stream = sharpCompressStream.BaseStream(); + } + + if (stream is SeekableSharpCompressStream seekableSharpCompressStream) + { + stream = seekableSharpCompressStream.BaseStream(); + } + + return stream is SharpCompressStream ? null + : stream.CanSeek ? stream + : null; + } + + private static Stream? GetPhysicalSeekableStream(Stream stream) + { + while (stream is SharpCompressStream sharpCompressStream) + { + var baseStream = sharpCompressStream.BaseStream(); + if (ReferenceEquals(baseStream, stream) || !baseStream.CanSeek) + { + break; + } + + stream = baseStream; + } + + return stream.CanSeek ? stream : null; + } + + private static bool IsProbeWrapper(Stream stream) => + stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream + && sharpCompressStream.BaseStream() is SharpCompressStream { IsPassthrough: false }; + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (Mode != CompressionMode.Decompress) + { + return; + } + + if (read > 0) + { + UpdateChecksum(buffer); + return; + } + + ValidateTrailer(); + } + + private void UpdateChecksum(ReadOnlySpan buffer) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table.NotNull(), _seed, buffer); + _readCount += (ulong)buffer.Length; + } + + private void ValidateTrailer() + { + if (_trailerValidated || _skipTrailerValidation || Mode != CompressionMode.Decompress) + { + return; + } + + _trailerValidated = true; + + var countingStream = _countingReadableSubStream.NotNull(); + ulong? compressedDataSize = null; + Span trailer = stackalloc byte[20]; + if (_expectedMemberSize.HasValue && countingStream.CanSeek) + { + compressedDataSize = _expectedMemberSize.Value - 26; + countingStream.Position = _compressedDataStartPosition + (long)compressedDataSize.Value; + countingStream.ReadFully(trailer); + } + else if (GetPhysicalSeekableStream(countingStream.WrappedStream) is { } trailerStream) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 20; + trailerStream.ReadFully(trailer); + trailerStream.Position = position; + } + else + { + compressedDataSize = _stream is LzmaStream lzmaStream + ? (ulong)lzmaStream.CompressedBytesRead + : (ulong)countingStream.BytesRead; + if (countingStream.CanSeek) + { + countingStream.Position = + _compressedDataStartPosition + (long)compressedDataSize.Value; + } + countingStream.ReadFully(trailer); + } + + var expectedCrc = BinaryPrimitives.ReadUInt32LittleEndian(trailer); + var expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[4..]); + var expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[12..]); + + var actualCrc = ~_seed; + if (actualCrc != expectedCrc) + { + throw new InvalidFormatException( + $"LZip CRC mismatch. Expected 0x{expectedCrc:X8}, actual 0x{actualCrc:X8}." + ); + } + + if (_readCount != expectedDataSize) + { + throw new InvalidFormatException( + $"LZip data size mismatch. Expected {expectedDataSize}, actual {_readCount}." + ); + } + + var actualMemberSize = compressedDataSize ?? expectedMemberSize - 26; + actualMemberSize += 26; + if (actualMemberSize != expectedMemberSize) + { + throw new InvalidFormatException( + $"LZip member size mismatch. Expected {expectedMemberSize}, actual {actualMemberSize}." + ); + } + } } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index bdc189ff..02fd16d7 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -11,6 +11,8 @@ namespace SharpCompress.Compressors.LZMA; public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { + internal bool HasEndMarker => _rep0 == uint.MaxValue; + public void Dispose() { _outWindow?.Dispose(); diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 4ad30127..c89fb923 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -92,6 +92,11 @@ public partial class LzmaStream if (control == 0x00) { + if (_isLzma2 && _decoder is { HasEndMarker: true }) + { + throw new DataErrorException(); + } + _endReached = true; return; } @@ -213,10 +218,9 @@ public partial class LzmaStream await _decoder! .CodeAsync(_dictionarySize, _outWindow, _rangeDecoder, cancellationToken) .ConfigureAwait(false) - && _outputSize < 0 ) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -227,6 +231,11 @@ public partial class LzmaStream if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + if ( !_rangeDecoder.IsFinished || (_rangeDecoderLimit >= 0 && _rangeDecoder._total != _rangeDecoderLimit) @@ -325,10 +334,9 @@ public partial class LzmaStream await _decoder! .CodeAsync(_dictionarySize, _outWindow, _rangeDecoder, cancellationToken) .ConfigureAwait(false) - && _outputSize < 0 ) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -339,6 +347,11 @@ public partial class LzmaStream if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + if ( !_rangeDecoder.IsFinished || (_rangeDecoderLimit >= 0 && _rangeDecoder._total != _rangeDecoderLimit) diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 38bb2eca..f45b33a3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -277,9 +277,9 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { _inputPosition += _outWindow.CopyStream(_inputStream, toProcess); } - else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) + else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder)) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -290,6 +290,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + // Check range corruption scenario if ( !_rangeDecoder.IsFinished @@ -368,9 +373,9 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { _inputPosition += _outWindow.CopyStream(_inputStream, 1); } - else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) + else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder)) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var value = _outWindow.ReadByte(); @@ -379,6 +384,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + // Check range corruption scenario if ( !_rangeDecoder.IsFinished @@ -413,6 +423,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (control == 0x00) { + if (_isLzma2 && _decoder is { HasEndMarker: true }) + { + throw new DataErrorException(); + } + _endReached = true; return; } @@ -472,6 +487,19 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } } + private void HandleEndMarker() + { + if (_isLzma2) + { + throw new DataErrorException(); + } + + if (_outputSize < 0) + { + _availableBytes = _outWindow.AvailableBytes; + } + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); @@ -485,4 +513,6 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } public byte[] Properties { get; } = new byte[5]; + + internal long CompressedBytesRead => _inputPosition; } diff --git a/src/SharpCompress/Compressors/Xz/Crc32.cs b/src/SharpCompress/Compressors/Xz/Crc32.cs index be303898..acea8ecd 100644 --- a/src/SharpCompress/Compressors/Xz/Crc32.cs +++ b/src/SharpCompress/Compressors/Xz/Crc32.cs @@ -52,6 +52,9 @@ public static class Crc32 return createTable; } + public static uint Update(uint seed, ReadOnlySpan buffer) => + CalculateHash(InitializeTable(DefaultPolynomial), seed, buffer); + private static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) { var crc = seed; diff --git a/src/SharpCompress/Compressors/Xz/Crc64.cs b/src/SharpCompress/Compressors/Xz/Crc64.cs index e37d04a9..3cb7fbe1 100644 --- a/src/SharpCompress/Compressors/Xz/Crc64.cs +++ b/src/SharpCompress/Compressors/Xz/Crc64.cs @@ -6,10 +6,13 @@ namespace SharpCompress.Compressors.Xz; public static class Crc64 { public const ulong DefaultSeed = 0x0; + internal const ulong XZ_SEED = 0xffffffffffffffff; internal static ulong[]? Table; + private static ulong[]? _xzTable; public const ulong Iso3309Polynomial = 0xD800000000000000; + private const ulong XZ_POLYNOMIAL = 0xC96C5795D7870F42; public static ulong Compute(byte[] buffer) => Compute(DefaultSeed, buffer); @@ -20,6 +23,15 @@ public static class Crc64 return CalculateHash(seed, Table, buffer); } + public static ulong ComputeXz(byte[] buffer) => ~UpdateXz(XZ_SEED, buffer); + + public static ulong UpdateXz(ulong seed, ReadOnlySpan buffer) + { + _xzTable ??= CreateTable(XZ_POLYNOMIAL); + + return CalculateHash(seed, _xzTable, buffer); + } + public static ulong CalculateHash(ulong seed, ulong[] table, ReadOnlySpan buffer) { var crc = seed; diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs index 7c6ab8ed..cac658a0 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs @@ -1,11 +1,10 @@ using System; -using System.Collections.Generic; +using System.Buffers; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; -using SharpCompress.Compressors.Xz.Filters; namespace SharpCompress.Compressors.Xz; @@ -32,8 +31,10 @@ public sealed partial class XZBlock if (!_endOfStream) { bytesRead = await _decomStream + .NotNull() .ReadAsync(buffer, offset, count, cancellationToken) .ConfigureAwait(false); + UpdateCheck(buffer, offset, bytesRead); } if (bytesRead != count) @@ -59,13 +60,24 @@ public sealed partial class XZBlock var bytes = (BaseStream.Position - _startPosition) % 4; if (bytes > 0) { - var paddingBytes = new byte[4 - bytes]; - await BaseStream - .ReadAsync(paddingBytes, 0, paddingBytes.Length, cancellationToken) - .ConfigureAwait(false); - if (paddingBytes.Any(b => b != 0)) + var size = 4 - (int)bytes; + var paddingBytes = ArrayPool.Shared.Rent(size); + try { - throw new InvalidFormatException("Padding bytes were non-null"); + await BaseStream + .ReadExactAsync(paddingBytes, 0, size, cancellationToken) + .ConfigureAwait(false); + for (var i = 0; i < size; i++) + { + if (paddingBytes[i] != 0) + { + throw new InvalidFormatException("Padding bytes were non-null"); + } + } + } + finally + { + ArrayPool.Shared.Return(paddingBytes); } } _paddingSkipped = true; @@ -73,11 +85,19 @@ public sealed partial class XZBlock private async ValueTask CheckCrcAsync(CancellationToken cancellationToken = default) { - var crc = new byte[_checkSize]; - await BaseStream.ReadAsync(crc, 0, _checkSize, cancellationToken).ConfigureAwait(false); - // Actually do a check (and read in the bytes - // into the function throughout the stream read). - _crcChecked = true; + var crc = ArrayPool.Shared.Rent(_checkSize); + try + { + await BaseStream + .ReadExactAsync(crc, 0, _checkSize, cancellationToken) + .ConfigureAwait(false); + VerifyCheck(crc.AsSpan().Slice(0, _checkSize)); + _crcChecked = true; + } + finally + { + ArrayPool.Shared.Return(crc); + } } private async ValueTask LoadHeaderAsync(CancellationToken cancellationToken = default) @@ -97,12 +117,19 @@ public sealed partial class XZBlock private async ValueTask ReadHeaderSizeAsync(CancellationToken cancellationToken = default) { - var buffer = new byte[1]; - await BaseStream.ReadAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false); - _blockHeaderSizeByte = buffer[0]; - if (_blockHeaderSizeByte == 0) + var buffer = ArrayPool.Shared.Rent(1); + try { - throw new XZIndexMarkerReachedException(); + await BaseStream.ReadExactAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false); + _blockHeaderSizeByte = buffer[0]; + if (_blockHeaderSizeByte == 0) + { + throw new XZIndexMarkerReachedException(); + } + } + finally + { + ArrayPool.Shared.Return(buffer); } } diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs index 00c967f7..81db843b 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs @@ -1,11 +1,10 @@ -#nullable disable - using System; +using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System.Security.Cryptography; using SharpCompress.Common; using SharpCompress.Compressors.Xz.Filters; @@ -14,16 +13,20 @@ namespace SharpCompress.Compressors.Xz; [CLSCompliant(false)] public sealed partial class XZBlock : XZReadOnlyStream { - public int BlockHeaderSize => (_blockHeaderSizeByte + 1) * 4; + private int BlockHeaderSize => (_blockHeaderSizeByte + 1) * 4; public ulong? CompressedSize { get; private set; } public ulong? UncompressedSize { get; private set; } - public Stack Filters { get; private set; } = new(); - public bool HeaderIsLoaded { get; private set; } + private readonly Stack _filters = new(); + private bool HeaderIsLoaded { get; set; } + private readonly CheckType _checkType; private readonly int _checkSize; + private uint _crc32 = Crc32.DefaultSeed; + private ulong _crc64 = Crc64.XZ_SEED; + private readonly SHA256? _sha256; private bool _streamConnected; private int _numFilters; private byte _blockHeaderSizeByte; - private Stream _decomStream; + private Stream? _decomStream; private bool _endOfStream; private bool _paddingSkipped; private bool _crcChecked; @@ -32,7 +35,12 @@ public sealed partial class XZBlock : XZReadOnlyStream public XZBlock(Stream stream, CheckType checkType, int checkSize) : base(stream) { + _checkType = checkType; _checkSize = checkSize; + if (checkType == CheckType.SHA256) + { + _sha256 = SHA256.Create(); + } _startPosition = stream.Position; } @@ -51,7 +59,8 @@ public sealed partial class XZBlock : XZReadOnlyStream if (!_endOfStream) { - bytesRead = _decomStream.Read(buffer, offset, count); + bytesRead = _decomStream.NotNull().Read(buffer, offset, count); + UpdateCheck(buffer, offset, bytesRead); } if (bytesRead != count) @@ -89,19 +98,110 @@ public sealed partial class XZBlock : XZReadOnlyStream private void CheckCrc() { - var crc = new byte[_checkSize]; - BaseStream.Read(crc, 0, _checkSize); - // Actually do a check (and read in the bytes - // into the function throughout the stream read). - _crcChecked = true; + var crc = ArrayPool.Shared.Rent(_checkSize); + try + { + BaseStream.ReadExact(crc, 0, _checkSize); + VerifyCheck(crc.AsSpan().Slice(0, _checkSize)); + _crcChecked = true; + } + finally + { + ArrayPool.Shared.Return(crc); + } + } + + private void UpdateCheck(byte[] buffer, int offset, int count) + { + if (count == 0 || _checkType == CheckType.NONE) + { + return; + } + + var bytes = buffer.AsSpan(offset, count); + switch (_checkType) + { + case CheckType.CRC32: + _crc32 = Crc32.Update(_crc32, bytes); + break; + case CheckType.CRC64: + _crc64 = Crc64.UpdateXz(_crc64, bytes); + break; + case CheckType.SHA256: + _sha256.NotNull().TransformBlock(buffer, offset, count, null, 0); + break; + } + } + + private void VerifyCheck(ReadOnlySpan expected) + { + switch (_checkType) + { + case CheckType.NONE: + break; + case CheckType.CRC32: + GetLittleEndianBytes(~_crc32, expected); + break; + case CheckType.CRC64: + GetLittleEndianBytes(~_crc64, expected); + break; + case CheckType.SHA256: + FinalizeSha256Check(expected); + break; + default: + throw new InvalidFormatException("Unsupported XZ check type"); + } + } + + private static void GetLittleEndianBytes(uint value, ReadOnlySpan expected) + { + var bytes = ArrayPool.Shared.Rent(sizeof(uint)); + try + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + if (!expected.SequenceEqual(bytes.AsSpan().Slice(0, sizeof(uint)))) + { + throw new InvalidFormatException("Block check corrupt"); + } + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private static void GetLittleEndianBytes(ulong value, ReadOnlySpan expected) + { + var bytes = ArrayPool.Shared.Rent(sizeof(ulong)); + try + { + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + if (!expected.SequenceEqual(bytes.AsSpan().Slice(0, sizeof(ulong)))) + { + throw new InvalidFormatException("Block check corrupt"); + } + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private void FinalizeSha256Check(ReadOnlySpan expected) + { + _sha256.NotNull().TransformFinalBlock(Array.Empty(), 0, 0); + if (!expected.SequenceEqual(_sha256.NotNull().Hash)) + { + throw new InvalidFormatException("Block check corrupt"); + } } private void ConnectStream() { _decomStream = BaseStream; - while (Filters.Any()) + while (_filters.Any()) { - var filter = Filters.Pop(); + var filter = _filters.Pop(); filter.SetBaseStream(_decomStream); _decomStream = filter; } @@ -199,7 +299,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } filter.ValidateFilter(); - Filters.Push(filter); + _filters.Push(filter); } if (nonLastSizeChangers > 2) { diff --git a/src/SharpCompress/Crypto/Crc32Stream.cs b/src/SharpCompress/Crypto/Crc32Stream.cs index b3294831..67dcb913 100644 --- a/src/SharpCompress/Crypto/Crc32Stream.cs +++ b/src/SharpCompress/Crypto/Crc32Stream.cs @@ -119,7 +119,7 @@ public sealed class Crc32Stream : Stream public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer) => ~CalculateCrc(InitializeTable(polynomial), seed, buffer); - private static uint[] InitializeTable(uint polynomial) + internal static uint[] InitializeTable(uint polynomial) { if (polynomial == DEFAULT_POLYNOMIAL && _defaultTable != null) { @@ -153,7 +153,7 @@ public sealed class Crc32Stream : Stream return createTable; } - private static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan buffer) + internal static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan buffer) { unchecked { @@ -165,6 +165,6 @@ public sealed class Crc32Stream : Stream return crc; } - private static uint CalculateCrc(uint[] table, uint crc, byte b) => + internal static uint CalculateCrc(uint[] table, uint crc, byte b) => (crc >> 8) ^ table[(crc ^ b) & 0xFF]; } diff --git a/src/SharpCompress/IO/CountingStream.cs b/src/SharpCompress/IO/CountingStream.cs index 5d1263b3..ef545c9c 100644 --- a/src/SharpCompress/IO/CountingStream.cs +++ b/src/SharpCompress/IO/CountingStream.cs @@ -6,11 +6,12 @@ using System.Threading.Tasks; namespace SharpCompress.IO; /// -/// A simple stream wrapper that counts bytes written without buffering. +/// A simple stream wrapper that counts bytes read and written without buffering. /// internal class CountingStream : Stream { private readonly Stream _stream; + private long _bytesRead; private long _bytesWritten; public CountingStream(Stream stream) @@ -18,6 +19,13 @@ internal class CountingStream : Stream _stream = stream ?? throw new ArgumentNullException(nameof(stream)); } + internal Stream WrappedStream => _stream; + + /// + /// Gets the total number of bytes read from this stream. + /// + public long BytesRead => _bytesRead; + /// /// Gets the total number of bytes written to this stream. /// @@ -42,8 +50,32 @@ internal class CountingStream : Stream public override async Task FlushAsync(CancellationToken cancellationToken) => await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + _bytesRead += read; + return read; + } + + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value != -1) + { + _bytesRead++; + } + + return value; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + _bytesRead += read; + return read; + } +#endif public override long Seek(long offset, SeekOrigin origin) => _stream.Seek(offset, origin); @@ -72,7 +104,31 @@ internal class CountingStream : Stream _bytesWritten += count; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + _bytesRead += read; + return read; + } + #if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _bytesRead += read; + return read; + } + public override async ValueTask WriteAsync( ReadOnlyMemory buffer, CancellationToken cancellationToken = default diff --git a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs index a982982c..7515b257 100644 --- a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs +++ b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs @@ -36,7 +36,9 @@ public static class IAsyncReaderExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await reader .Entry.WriteEntryToFileAsync( destinationFileName, @@ -44,12 +46,12 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) - .ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options, ct).ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } /// /// Extract all remaining unread entries to specific directory asynchronously, retaining filename @@ -72,7 +74,9 @@ public static class IAsyncReaderExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await reader .Entry.WriteEntryToFileAsync( destinationFileName, @@ -80,12 +84,12 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) - .ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options, ct).ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } public async ValueTask WriteEntryToAsync( FileInfo destinationFileInfo, @@ -100,7 +104,7 @@ public static class IAsyncReaderExtensions private static async ValueTask CopyEntryToAsync( IAsyncReader reader, Stream writableStream, - int? bufferSize, + ExtractionOptions options, CancellationToken cancellationToken ) { @@ -113,9 +117,14 @@ public static class IAsyncReaderExtensions .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); #endif - var sourceStream = WrapWithProgress(entryStream, reader.Entry); + var checkedStream = IEntryExtensions.WrapWithChecksumValidation( + reader.Entry, + entryStream, + options + ); + var sourceStream = WrapWithProgress(checkedStream, reader.Entry); await sourceStream - .CopyToAsync(writableStream, bufferSize ?? Constants.BufferSize, cancellationToken) + .CopyToAsync(writableStream, options.BufferSize, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Readers/IReaderExtensions.cs b/src/SharpCompress/Readers/IReaderExtensions.cs index 93c391d2..8eb97193 100644 --- a/src/SharpCompress/Readers/IReaderExtensions.cs +++ b/src/SharpCompress/Readers/IReaderExtensions.cs @@ -51,26 +51,35 @@ public static class IReaderExtensions /// /// Extract to specific file /// - public void WriteEntryToFile( - string destinationFileName, - ExtractionOptions? options = null - ) => + public void WriteEntryToFile(string destinationFileName, ExtractionOptions? options = null) + { + options ??= new ExtractionOptions(); reader.Entry.WriteEntryToFile( destinationFileName, options, (x, fm) => { using var fs = File.Open(x, fm); - CopyEntryTo(reader, fs, options?.BufferSize ?? Constants.BufferSize); + CopyEntryTo(reader, fs, options ?? new ExtractionOptions()); } ); + } } - private static void CopyEntryTo(IReader reader, Stream writableStream, int bufferSize) + private static void CopyEntryTo( + IReader reader, + Stream writableStream, + ExtractionOptions options + ) { using var entryStream = reader.OpenEntryStream(); - var sourceStream = WrapWithProgress(entryStream, reader.Entry); - sourceStream.CopyTo(writableStream, bufferSize); + var checkedStream = IEntryExtensions.WrapWithChecksumValidation( + reader.Entry, + entryStream, + options + ); + var sourceStream = WrapWithProgress(checkedStream, reader.Entry); + sourceStream.CopyTo(writableStream, options.BufferSize); } private static Stream WrapWithProgress(Stream source, IEntry entry) diff --git a/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs b/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs new file mode 100644 index 00000000..9bd7f44a --- /dev/null +++ b/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs @@ -0,0 +1,45 @@ +using System.IO; +using System.Text; +using SharpCompress.Common; +using SharpCompress.Compressors.BZip2; +using Xunit; + +namespace SharpCompress.Test.BZip2; + +public class BZip2StreamTests +{ + [Fact] + public void BZip2Stream_Throws_On_Corrupt_Checksum() + { + var compressed = Compress("BZip2 checksum validation test data."); + compressed[^5] ^= 1; + + using var stream = BZip2Stream.Create( + new MemoryStream(compressed), + SharpCompress.Compressors.CompressionMode.Decompress, + false + ); + using var output = new MemoryStream(); + + Assert.Throws(() => stream.CopyTo(output)); + } + + private static byte[] Compress(string value) + { + using var memoryStream = new MemoryStream(); + using ( + var bzip2Stream = BZip2Stream.Create( + memoryStream, + SharpCompress.Compressors.CompressionMode.Compress, + false, + leaveOpen: true + ) + ) + { + var bytes = Encoding.ASCII.GetBytes(value); + bzip2Stream.Write(bytes, 0, bytes.Length); + } + + return memoryStream.ToArray(); + } +} diff --git a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs new file mode 100644 index 00000000..2e98c962 --- /dev/null +++ b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.GZip; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Readers.GZip; +using SharpCompress.Test.Mocks; +using Xunit; + +namespace SharpCompress.Test.GZip; + +public class GZipCrcExtractionTests : TestBase +{ + [Fact] + public void GZipArchive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + [Fact] + public void GZipArchive_WriteToFile_Throws_On_Size_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: false)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + [Fact] + public void GZipReader_WriteEntryToFile_Throws_On_NonSeekable_Crc_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var nonSeekableStream = new ForwardOnlyStream(stream); + using var reader = GZipReader.OpenReader(nonSeekableStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.True(reader.MoveToNextEntry()); + Assert.Throws(() => reader.WriteEntryToFile(destination)); + } + + [Fact] + public void GZipArchive_WriteToFile_Skips_Trailer_Validation_When_CheckCrc_Is_False() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal( + new FileInfo(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar")).Length, + new FileInfo(destination).Length + ); + } + + [Fact] + public async Task GZipArchive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { +#if LEGACY_DOTNET + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#else + await using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#endif + await using var archive = await GZipArchive.OpenAsyncArchive(stream); + var entry = await archive.EntriesAsync.SingleAsync(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + } + + private static byte[] ReadCorruptedGZipTrailer(bool corruptCrc) + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz")); + var trailer = bytes.AsSpan(bytes.Length - 8); + var offset = corruptCrc ? 0 : 4; + var value = BinaryPrimitives.ReadUInt32LittleEndian(trailer[offset..]); + BinaryPrimitives.WriteUInt32LittleEndian(trailer[offset..], value + 1); + return bytes; + } +} diff --git a/tests/SharpCompress.Test/OptionsUsabilityTests.cs b/tests/SharpCompress.Test/OptionsUsabilityTests.cs index 8b08b265..1902e4ab 100644 --- a/tests/SharpCompress.Test/OptionsUsabilityTests.cs +++ b/tests/SharpCompress.Test/OptionsUsabilityTests.cs @@ -306,6 +306,7 @@ public class OptionsUsabilityTests : TestBase Assert.True(preserveMetadata.PreserveAttributes); Assert.Equal(Constants.BufferSize, new ExtractionOptions().BufferSize); + Assert.True(new ExtractionOptions().CheckCrc); } [Fact] diff --git a/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs b/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs new file mode 100644 index 00000000..f2dc8f4f --- /dev/null +++ b/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs @@ -0,0 +1,36 @@ +using System.IO; +using System.Linq; +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Common; +using Xunit; + +namespace SharpCompress.Test.Rar; + +public class RarCrcExtractionTests : ArchiveTests +{ + [Theory] + [InlineData("Rar.rar")] + [InlineData("Rar5.rar")] + public void Rar_Archive_WriteToFile_Throws_On_Crc_Mismatch(string archiveName) + { + using var archive = RarArchive.OpenArchive(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, $"{archiveName}-crc-mismatch.txt"); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + private static RarArchiveEntry CorruptFirstFileCrc(IArchive archive) + { + var entry = archive.Entries.OfType().First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static void CorruptCrc(RarArchiveEntry entry) + { + var crc = entry.FileHeader.FileCrc.NotNull(); + crc[0] ^= 0xFF; + } +} diff --git a/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs new file mode 100644 index 00000000..45034123 --- /dev/null +++ b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test; + +public class RemainingCrcExtractionTests : TestBase +{ + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Throws_On_Checksum_Mismatch( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + Assert.Throws(() => reader.WriteAllToDirectory(destination)); + } + + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Skips_Checksum_When_CheckCrc_Is_False( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + reader.WriteAllToDirectory(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.True(Directory.GetFiles(destination, "*", SearchOption.AllDirectories).Length > 0); + } + + [Fact] + public void LZipStream_Throws_On_Trailer_Crc_Mismatch() + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.lz")); + bytes[^20] ^= 1; + using var stream = LZipStream.Create( + new MemoryStream(bytes), + SharpCompress.Compressors.CompressionMode.Decompress + ); + using var output = new MemoryStream(); + + Assert.Throws(() => stream.CopyTo(output)); + } + + private static byte[] ReadCorruptedArchive(string archiveName, string payloadMarker) + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + var marker = System.Text.Encoding.ASCII.GetBytes(payloadMarker); + var offset = bytes.AsSpan().IndexOf(marker); + if (offset < 0) + { + throw new InvalidOperationException($"Payload marker '{payloadMarker}' was not found."); + } + + bytes[offset] ^= 1; + return bytes; + } +} diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs new file mode 100644 index 00000000..3261f16a --- /dev/null +++ b/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs @@ -0,0 +1,81 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Common; +using SharpCompress.Common.SevenZip; +using Xunit; + +namespace SharpCompress.Test.SevenZip; + +public class SevenZipCrcExtractionTests : ArchiveTests +{ + [Fact] + public void SevenZip_Archive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var archive = SevenZipArchive.OpenArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(entry.Key!, exception.Message); + } + + [Fact] + public void SevenZip_Archive_WriteToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var archive = SevenZipArchive.OpenArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-disabled.txt"); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.True(new FileInfo(destination).Length > 0); + } + + [Fact] + public async Task SevenZip_Archive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { + await using var archive = await SevenZipArchive.OpenAsyncArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entries = await archive.EntriesAsync.ToListAsync(); + var entry = CorruptFirstFileCrc(entries); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-mismatch-async.txt"); + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + + Assert.Contains(entry.Key!, exception.Message); + } + + private static IArchiveEntry CorruptFirstFileCrc(IArchive archive) + { + var entry = archive.Entries.First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static IArchiveEntry CorruptFirstFileCrc( + System.Collections.Generic.IEnumerable entries + ) + { + var entry = entries.First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static void CorruptCrc(IArchiveEntry entry) + { + var sevenZipEntry = Assert.IsAssignableFrom(entry); + var crc = sevenZipEntry.FilePart.Header.Crc.NotNull(); + sevenZipEntry.FilePart.Header.Crc = crc ^ 0xFFFFFFFF; + } +} diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index 9f576d9a..cc41e831 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -2,14 +2,27 @@ using System; using System.Buffers; using System.IO; using System.Threading.Tasks; +using SharpCompress.Common; using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.Xz; using SharpCompress.Test.Mocks; using Xunit; namespace SharpCompress.Test.Streams; -public class LzmaStreamAsyncTests +public class LzmaStreamAsyncTests : TestBase { + [Fact] + public async ValueTask TestLzma2Decompress() + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "bad-1-lzma2-7.xz")); + + using var xz = new XZStream(stream); + await Assert.ThrowsAnyAsync(async () => + await xz.TransferToAsync(Stream.Null, long.MaxValue) + ); + } + [Fact] public async ValueTask TestLzma2Decompress1ByteAsync() { diff --git a/tests/SharpCompress.Test/Xz/Crc64Tests.cs b/tests/SharpCompress.Test/Xz/Crc64Tests.cs index 8cf71e1e..e2338c64 100644 --- a/tests/SharpCompress.Test/Xz/Crc64Tests.cs +++ b/tests/SharpCompress.Test/Xz/Crc64Tests.cs @@ -27,4 +27,12 @@ public class Crc64Tests Assert.Equal((ulong)0x416B4150508661EE, actual); } + + [Fact] + public void XzCheckString() + { + var actual = Crc64.ComputeXz(Encoding.ASCII.GetBytes("123456789")); + + Assert.Equal(0x995DC9BBDF1939FAUL, actual); + } } diff --git a/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs b/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs index 6460ed53..9d59d6d4 100644 --- a/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs +++ b/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs @@ -85,7 +85,7 @@ public class XzBlockAsyncTests : XzTestsBase { using var xzBlock = new XZBlock(CompressedStream, CheckType.CRC64, 8); using var sr = new StreamReader(xzBlock); - Assert.Equal(await sr.ReadToEndAsync().ConfigureAwait(false), Original); + Assert.Equal(Original, await sr.ReadToEndAsync().ConfigureAwait(false)); } [Fact] @@ -101,8 +101,8 @@ public class XzBlockAsyncTests : XzTestsBase [Fact] public async ValueTask SkipsPaddingWhenPresentAsync() { - // CompressedIndexedStream's first block has 1-byte padding. - using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC64, 8); + // CompressedIndexedStream uses CRC32 checks. + using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC32, 4); using var sr = new StreamReader(xzBlock); await sr.ReadToEndAsync().ConfigureAwait(false); Assert.Equal(0L, CompressedIndexedStream.Position % 4L); diff --git a/tests/SharpCompress.Test/Xz/XZBlockTests.cs b/tests/SharpCompress.Test/Xz/XZBlockTests.cs index 3b06008e..9b6efe6e 100644 --- a/tests/SharpCompress.Test/Xz/XZBlockTests.cs +++ b/tests/SharpCompress.Test/Xz/XZBlockTests.cs @@ -72,7 +72,7 @@ public class XzBlockTests : XzTestsBase { var xzBlock = new XZBlock(CompressedStream, CheckType.CRC64, 8); using var sr = new StreamReader(xzBlock); - Assert.Equal(sr.ReadToEnd(), Original); + Assert.Equal(Original, sr.ReadToEnd()); } [Fact] @@ -88,8 +88,8 @@ public class XzBlockTests : XzTestsBase [Fact] public void SkipsPaddingWhenPresent() { - // CompressedIndexedStream's first block has 1-byte padding. - using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC64, 8); + // CompressedIndexedStream uses CRC32 checks. + using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC32, 4); using var sr = new StreamReader(xzBlock); sr.ReadToEnd(); Assert.Equal(0L, CompressedIndexedStream.Position % 4L); diff --git a/tests/SharpCompress.Test/Xz/XZStreamTests.cs b/tests/SharpCompress.Test/Xz/XZStreamTests.cs index 80ad5dc3..00cd2f27 100644 --- a/tests/SharpCompress.Test/Xz/XZStreamTests.cs +++ b/tests/SharpCompress.Test/Xz/XZStreamTests.cs @@ -1,4 +1,5 @@ using System.IO; +using SharpCompress.Common; using SharpCompress.Compressors.Xz; using SharpCompress.IO; using SharpCompress.Test.Mocks; @@ -54,4 +55,15 @@ public class XzStreamTests : XzTestsBase var uncompressed = sr.ReadToEnd(); Assert.Equal(OriginalEmpty, uncompressed); } + + [Fact] + public void Throws_On_Corrupt_Block_Check() + { + var compressed = (byte[])Compressed.Clone(); + compressed[compressed.Length - 29] ^= 1; + using var xz = new XZStream(new MemoryStream(compressed)); + using var output = new MemoryStream(); + + Assert.Throws(() => xz.CopyTo(output)); + } } diff --git a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs new file mode 100644 index 00000000..cd0e610c --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs @@ -0,0 +1,260 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Writers; +using SharpCompress.Writers.Zip; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class ZipCrcExtractionTests : ArchiveTests +{ + private const string EntryName = "crc.txt"; + private static readonly byte[] EntryData = Encoding.UTF8.GetBytes("crc validation payload"); + + [Fact] + public void Zip_Archive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-disabled.txt"); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, File.ReadAllBytes(destination)); + } + + [Fact] + public void Zip_Archive_WriteTo_Throws_On_Crc_Mismatch_When_Enabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + using var destination = new MemoryStream(); + + var exception = Assert.Throws(() => + entry.WriteTo(destination, new ExtractionOptions { CheckCrc = true }) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteTo_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + using var destination = new MemoryStream(); + + entry.WriteTo(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, destination.ToArray()); + } + + [Fact] + public void Zip_Reader_WriteEntryToFile_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var reader = ReaderFactory.OpenReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-mismatch.txt"); + + Assert.True(reader.MoveToNextEntry()); + var exception = Assert.Throws(() => + reader.WriteEntryToFile(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Reader_WriteEntryToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var reader = ReaderFactory.OpenReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-disabled.txt"); + + Assert.True(reader.MoveToNextEntry()); + reader.WriteEntryToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, File.ReadAllBytes(destination)); + } + + [Fact] + public async Task Zip_Archive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-mismatch-async.txt"); + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public async Task Zip_Archive_WriteToAsync_Throws_On_Crc_Mismatch_When_Enabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); +#if LEGACY_DOTNET + using var destination = new MemoryStream(); +#else + await using var destination = new MemoryStream(); +#endif + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = true }) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public async Task Zip_Archive_WriteToAsync_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); +#if LEGACY_DOTNET + using var destination = new MemoryStream(); +#else + await using var destination = new MemoryStream(); +#endif + + await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, destination.ToArray()); + } + + [Fact] + public async Task Zip_Reader_WriteEntryToFileAsync_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + await using var reader = await ReaderFactory.OpenAsyncReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-mismatch-async.txt"); + + Assert.True(await reader.MoveToNextEntryAsync()); + var exception = await Assert.ThrowsAsync(async () => + await reader.WriteEntryToFileAsync(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteToFile_Throws_On_DataDescriptor_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: true); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-dd-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(EntryName, exception.Message); + } + + private static MemoryStream CreateZipWithInvalidCrc(bool useDataDescriptor) + { + var zipStream = new MemoryStream(); + Stream writerStream = useDataDescriptor ? new NonSeekableWriteStream(zipStream) : zipStream; + using ( + var writer = WriterFactory.OpenWriter( + writerStream, + ArchiveType.Zip, + new ZipWriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true } + ) + ) + { + writer.Write(EntryName, new MemoryStream(EntryData)); + } + + var bytes = zipStream.ToArray(); + CorruptCrc(bytes, ZipHeaderFactoryEntrySignature, 14); + CorruptCrc(bytes, ZipHeaderFactoryDirectorySignature, 16); + return new MemoryStream(bytes); + } + + private const uint ZipHeaderFactoryEntrySignature = 0x04034b50; + private const uint ZipHeaderFactoryDirectorySignature = 0x02014b50; + + private static void CorruptCrc(byte[] bytes, uint signature, int crcOffset) + { + var offset = FindSignature(bytes, signature); + var crcIndex = offset + crcOffset; + bytes[crcIndex] ^= 0xFF; + } + + private static int FindSignature(byte[] bytes, uint signature) + { + var signatureBytes = BitConverter.GetBytes(signature); + for (var i = 0; i <= bytes.Length - signatureBytes.Length; i++) + { + if (bytes.AsSpan(i, signatureBytes.Length).SequenceEqual(signatureBytes)) + { + return i; + } + } + + throw new InvalidOperationException($"ZIP signature 0x{signature:X8} was not found."); + } + + private sealed class NonSeekableWriteStream(Stream stream) : Stream + { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() => stream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + stream.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + stream.Write(buffer, offset, count); + +#if !LEGACY_DOTNET + public override void Write(ReadOnlySpan buffer) => stream.Write(buffer); +#endif + } +} diff --git a/tests/TestArchives/Archives/bad-1-lzma2-7.xz b/tests/TestArchives/Archives/bad-1-lzma2-7.xz new file mode 100644 index 00000000..8cc711c1 Binary files /dev/null and b/tests/TestArchives/Archives/bad-1-lzma2-7.xz differ