From 4e3dcabbd02b491f05e5f366ba1d9e6051edb40d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 09:24:04 +0100 Subject: [PATCH 01/12] Add basic crc checking for xz --- src/SharpCompress/Compressors/Xz/Crc32.cs | 3 + .../Compressors/Xz/XZBlock.Async.cs | 8 +- src/SharpCompress/Compressors/Xz/XZBlock.cs | 77 +++++++++++++++++- .../Streams/LzmaStreamAsyncTests.cs | 15 +++- tests/TestArchives/Archives/bad-1-lzma2-7.xz | Bin 0 -> 408 bytes 5 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 tests/TestArchives/Archives/bad-1-lzma2-7.xz diff --git a/src/SharpCompress/Compressors/Xz/Crc32.cs b/src/SharpCompress/Compressors/Xz/Crc32.cs index be303898..acea8ecd 100644 --- a/src/SharpCompress/Compressors/Xz/Crc32.cs +++ b/src/SharpCompress/Compressors/Xz/Crc32.cs @@ -52,6 +52,9 @@ public static class Crc32 return createTable; } + public static uint Update(uint seed, ReadOnlySpan buffer) => + CalculateHash(InitializeTable(DefaultPolynomial), seed, buffer); + private static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) { var crc = seed; diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs index 7c6ab8ed..cc3c31fe 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs @@ -34,6 +34,7 @@ public sealed partial class XZBlock bytesRead = await _decomStream .ReadAsync(buffer, offset, count, cancellationToken) .ConfigureAwait(false); + UpdateCheck(buffer, offset, bytesRead); } if (bytesRead != count) @@ -74,9 +75,10 @@ public sealed partial class XZBlock private async ValueTask CheckCrcAsync(CancellationToken cancellationToken = default) { var crc = new byte[_checkSize]; - await BaseStream.ReadAsync(crc, 0, _checkSize, cancellationToken).ConfigureAwait(false); - // Actually do a check (and read in the bytes - // into the function throughout the stream read). + await BaseStream + .ReadExactAsync(crc, 0, _checkSize, cancellationToken) + .ConfigureAwait(false); + VerifyCheck(crc); _crcChecked = true; } diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs index 00c967f7..778f33b5 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs @@ -1,9 +1,11 @@ #nullable disable using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; @@ -19,7 +21,11 @@ public sealed partial class XZBlock : XZReadOnlyStream public ulong? UncompressedSize { get; private set; } public Stack Filters { get; private set; } = new(); public bool HeaderIsLoaded { get; private set; } + private readonly CheckType _checkType; private readonly int _checkSize; + private uint _crc32 = Crc32.DefaultSeed; + private ulong _crc64 = Crc64.DefaultSeed; + private readonly SHA256 _sha256; private bool _streamConnected; private int _numFilters; private byte _blockHeaderSizeByte; @@ -32,7 +38,12 @@ public sealed partial class XZBlock : XZReadOnlyStream public XZBlock(Stream stream, CheckType checkType, int checkSize) : base(stream) { + _checkType = checkType; _checkSize = checkSize; + if (checkType == CheckType.SHA256) + { + _sha256 = SHA256.Create(); + } _startPosition = stream.Position; } @@ -52,6 +63,7 @@ public sealed partial class XZBlock : XZReadOnlyStream if (!_endOfStream) { bytesRead = _decomStream.Read(buffer, offset, count); + UpdateCheck(buffer, offset, bytesRead); } if (bytesRead != count) @@ -90,12 +102,71 @@ public sealed partial class XZBlock : XZReadOnlyStream private void CheckCrc() { var crc = new byte[_checkSize]; - BaseStream.Read(crc, 0, _checkSize); - // Actually do a check (and read in the bytes - // into the function throughout the stream read). + BaseStream.ReadExact(crc, 0, _checkSize); + VerifyCheck(crc); _crcChecked = true; } + private void UpdateCheck(byte[] buffer, int offset, int count) + { + if (count == 0 || _checkType == CheckType.NONE) + { + return; + } + + var bytes = buffer.AsSpan(offset, count); + switch (_checkType) + { + case CheckType.CRC32: + _crc32 = Crc32.Update(_crc32, bytes); + break; + case CheckType.CRC64: + Crc64.Table ??= Crc64.CreateTable(Crc64.Iso3309Polynomial); + _crc64 = Crc64.CalculateHash(_crc64, Crc64.Table, bytes); + break; + case CheckType.SHA256: + _sha256.TransformBlock(buffer, offset, count, null, 0); + break; + } + } + + private void VerifyCheck(byte[] expected) + { + var actual = _checkType switch + { + CheckType.NONE => Array.Empty(), + CheckType.CRC32 => GetLittleEndianBytes(~_crc32), + CheckType.CRC64 => GetLittleEndianBytes(_crc64), + CheckType.SHA256 => FinalizeSha256Check(), + _ => throw new InvalidFormatException("Unsupported XZ check type"), + }; + + if (!expected.SequenceEqual(actual)) + { + throw new InvalidFormatException("Block check corrupt"); + } + } + + private static byte[] GetLittleEndianBytes(uint value) + { + var bytes = new byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return bytes; + } + + private static byte[] GetLittleEndianBytes(ulong value) + { + var bytes = new byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + return bytes; + } + + private byte[] FinalizeSha256Check() + { + _sha256.TransformFinalBlock(Array.Empty(), 0, 0); + return _sha256.Hash; + } + private void ConnectStream() { _decomStream = BaseStream; diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index 9f576d9a..5dd0348f 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -2,14 +2,27 @@ using System; using System.Buffers; using System.IO; using System.Threading.Tasks; +using SharpCompress.Common; using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.Xz; using SharpCompress.Test.Mocks; using Xunit; namespace SharpCompress.Test.Streams; -public class LzmaStreamAsyncTests +public class LzmaStreamAsyncTests : TestBase { + [Fact] + public async ValueTask TestLzma2Decompress() + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "bad-1-lzma2-7.xz")); + + var xz = new XZStream(stream); + await Assert.ThrowsAsync(async () => + await xz.TransferToAsync(Stream.Null, long.MaxValue) + ); + } + [Fact] public async ValueTask TestLzma2Decompress1ByteAsync() { diff --git a/tests/TestArchives/Archives/bad-1-lzma2-7.xz b/tests/TestArchives/Archives/bad-1-lzma2-7.xz new file mode 100644 index 0000000000000000000000000000000000000000..8cc711c19ed67c7416a811fe0fdf71210487c23a GIT binary patch literal 408 zcmexsUKJ6=z`*kC+7>q^21P~=1_p*3{K~=)7*8;U$1 z%}Z;9ma^WSuvcH?a-mR-?^(f>k5d`-+C1APIT-~_(G!T@-q-*4tMoB7#t)l!@6J0~ zHSOX36DEhh`Zu#_1U9DpY{)8fzwl_;HQp4%^IZl>zl%-11Xz!pd%xx1sCIYRT~o`bM`s(^{!)_rcotaXcOFS zI(gS4)mK|h*9G|PdvKR$SJ&YWD&pThJUMfFue$d3)d#00ZP?#1bp;1g{l#nA<-Ia_ z=a$~Q6?^$v_txi?j>|63t5x6QIkT1fBcH3>Z>{gw`D)I8_!8tHQ7!2Gap%g$m+D;R zheP*n3BA2``~AxKZd<2%S5^z&)D?cradlzK0!QmS!OLbpt3pEr?z27gxgPGhH!ZVP zdcniy3=Y`^e@)uv@g7u=PA$D_9bbIvQ1k!)FR}t}FfeRlcX=sp?A^k^_?hV>GXul@ U&l8k49!dYr#K6G7z!Dh+05S{0sQ>@~ literal 0 HcmV?d00001 From 59d6b82ac552ee7af1e628173656f0fb3377e8d6 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 09:37:21 +0100 Subject: [PATCH 02/12] Cleanup: use some arraypool buffers --- .../Compressors/Xz/XZBlock.Async.cs | 61 ++++++---- src/SharpCompress/Compressors/Xz/XZBlock.cs | 107 +++++++++++------- 2 files changed, 106 insertions(+), 62 deletions(-) diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs index cc3c31fe..0175faaf 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs @@ -1,11 +1,9 @@ -using System; -using System.Collections.Generic; +using System.Buffers; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; -using SharpCompress.Compressors.Xz.Filters; namespace SharpCompress.Compressors.Xz; @@ -32,6 +30,7 @@ public sealed partial class XZBlock if (!_endOfStream) { bytesRead = await _decomStream + .NotNull() .ReadAsync(buffer, offset, count, cancellationToken) .ConfigureAwait(false); UpdateCheck(buffer, offset, bytesRead); @@ -60,13 +59,21 @@ public sealed partial class XZBlock var bytes = (BaseStream.Position - _startPosition) % 4; if (bytes > 0) { - var paddingBytes = new byte[4 - bytes]; - await BaseStream - .ReadAsync(paddingBytes, 0, paddingBytes.Length, cancellationToken) - .ConfigureAwait(false); - if (paddingBytes.Any(b => b != 0)) + var size = 4 - (int)bytes; + var paddingBytes = ArrayPool.Shared.Rent(size); + try { - throw new InvalidFormatException("Padding bytes were non-null"); + await BaseStream + .ReadExactAsync(paddingBytes, 0, size, cancellationToken) + .ConfigureAwait(false); + if (paddingBytes.Any(b => b != 0)) + { + throw new InvalidFormatException("Padding bytes were non-null"); + } + } + finally + { + ArrayPool.Shared.Return(paddingBytes); } } _paddingSkipped = true; @@ -74,12 +81,19 @@ public sealed partial class XZBlock private async ValueTask CheckCrcAsync(CancellationToken cancellationToken = default) { - var crc = new byte[_checkSize]; - await BaseStream - .ReadExactAsync(crc, 0, _checkSize, cancellationToken) - .ConfigureAwait(false); - VerifyCheck(crc); - _crcChecked = true; + var crc = ArrayPool.Shared.Rent(_checkSize); + try + { + await BaseStream + .ReadExactAsync(crc, 0, _checkSize, cancellationToken) + .ConfigureAwait(false); + VerifyCheck(crc); + _crcChecked = true; + } + finally + { + ArrayPool.Shared.Return(crc); + } } private async ValueTask LoadHeaderAsync(CancellationToken cancellationToken = default) @@ -99,12 +113,19 @@ public sealed partial class XZBlock private async ValueTask ReadHeaderSizeAsync(CancellationToken cancellationToken = default) { - var buffer = new byte[1]; - await BaseStream.ReadAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false); - _blockHeaderSizeByte = buffer[0]; - if (_blockHeaderSizeByte == 0) + var buffer = ArrayPool.Shared.Rent(1); + try { - throw new XZIndexMarkerReachedException(); + await BaseStream.ReadExactAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false); + _blockHeaderSizeByte = buffer[0]; + if (_blockHeaderSizeByte == 0) + { + throw new XZIndexMarkerReachedException(); + } + } + finally + { + ArrayPool.Shared.Return(buffer); } } diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs index 778f33b5..4221e3c4 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs @@ -1,13 +1,10 @@ -#nullable disable - using System; +using System.Buffers; using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Compressors.Xz.Filters; @@ -16,20 +13,20 @@ namespace SharpCompress.Compressors.Xz; [CLSCompliant(false)] public sealed partial class XZBlock : XZReadOnlyStream { - public int BlockHeaderSize => (_blockHeaderSizeByte + 1) * 4; + private int BlockHeaderSize => (_blockHeaderSizeByte + 1) * 4; public ulong? CompressedSize { get; private set; } public ulong? UncompressedSize { get; private set; } - public Stack Filters { get; private set; } = new(); - public bool HeaderIsLoaded { get; private set; } + private readonly Stack _filters = new(); + private bool HeaderIsLoaded { get; set; } private readonly CheckType _checkType; private readonly int _checkSize; private uint _crc32 = Crc32.DefaultSeed; private ulong _crc64 = Crc64.DefaultSeed; - private readonly SHA256 _sha256; + private readonly SHA256? _sha256; private bool _streamConnected; private int _numFilters; private byte _blockHeaderSizeByte; - private Stream _decomStream; + private Stream? _decomStream; private bool _endOfStream; private bool _paddingSkipped; private bool _crcChecked; @@ -62,7 +59,7 @@ public sealed partial class XZBlock : XZReadOnlyStream if (!_endOfStream) { - bytesRead = _decomStream.Read(buffer, offset, count); + bytesRead = _decomStream.NotNull().Read(buffer, offset, count); UpdateCheck(buffer, offset, bytesRead); } @@ -125,54 +122,80 @@ public sealed partial class XZBlock : XZReadOnlyStream _crc64 = Crc64.CalculateHash(_crc64, Crc64.Table, bytes); break; case CheckType.SHA256: - _sha256.TransformBlock(buffer, offset, count, null, 0); + _sha256.NotNull().TransformBlock(buffer, offset, count, null, 0); break; } } private void VerifyCheck(byte[] expected) { - var actual = _checkType switch + switch (_checkType) { - CheckType.NONE => Array.Empty(), - CheckType.CRC32 => GetLittleEndianBytes(~_crc32), - CheckType.CRC64 => GetLittleEndianBytes(_crc64), - CheckType.SHA256 => FinalizeSha256Check(), - _ => throw new InvalidFormatException("Unsupported XZ check type"), - }; + case CheckType.NONE: + break; + case CheckType.CRC32: + GetLittleEndianBytes(~_crc32, expected); + break; + case CheckType.CRC64: + GetLittleEndianBytes(~_crc64, expected); + break; + case CheckType.SHA256: + FinalizeSha256Check(expected); + break; + default: + throw new InvalidFormatException("Unsupported XZ check type"); + } + } - if (!expected.SequenceEqual(actual)) + private static void GetLittleEndianBytes(uint value, byte[] expected) + { + var bytes = ArrayPool.Shared.Rent(sizeof(uint)); + try + { + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + if (!expected.SequenceEqual(bytes.AsSpan().Slice(0, sizeof(uint)))) + { + throw new InvalidFormatException("Block check corrupt"); + } + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private static void GetLittleEndianBytes(ulong value, byte[] expected) + { + var bytes = ArrayPool.Shared.Rent(sizeof(ulong)); + try + { + BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); + if (!expected.SequenceEqual(bytes.AsSpan().Slice(0, sizeof(ulong)))) + { + throw new InvalidFormatException("Block check corrupt"); + } + } + finally + { + ArrayPool.Shared.Return(bytes); + } + } + + private void FinalizeSha256Check(byte[] expected) + { + _sha256.NotNull().TransformFinalBlock(Array.Empty(), 0, 0); + if (!expected.SequenceEqual(_sha256.NotNull().Hash)) { throw new InvalidFormatException("Block check corrupt"); } } - private static byte[] GetLittleEndianBytes(uint value) - { - var bytes = new byte[sizeof(uint)]; - BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); - return bytes; - } - - private static byte[] GetLittleEndianBytes(ulong value) - { - var bytes = new byte[sizeof(ulong)]; - BinaryPrimitives.WriteUInt64LittleEndian(bytes, value); - return bytes; - } - - private byte[] FinalizeSha256Check() - { - _sha256.TransformFinalBlock(Array.Empty(), 0, 0); - return _sha256.Hash; - } - private void ConnectStream() { _decomStream = BaseStream; - while (Filters.Any()) + while (_filters.Any()) { - var filter = Filters.Pop(); + var filter = _filters.Pop(); filter.SetBaseStream(_decomStream); _decomStream = filter; } @@ -270,7 +293,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } filter.ValidateFilter(); - Filters.Push(filter); + _filters.Push(filter); } if (nonLastSizeChangers > 2) { From 160ba0938a23f6ec9462794deb478fd613ea80c5 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 10:00:39 +0100 Subject: [PATCH 03/12] review fixes --- .../Compressors/Xz/XZBlock.Async.cs | 10 +++++--- src/SharpCompress/Compressors/Xz/XZBlock.cs | 23 ++++++++++++------- .../Streams/LzmaStreamAsyncTests.cs | 2 +- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs index 0175faaf..cac658a0 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.Async.cs @@ -1,3 +1,4 @@ +using System; using System.Buffers; using System.IO; using System.Linq; @@ -66,9 +67,12 @@ public sealed partial class XZBlock await BaseStream .ReadExactAsync(paddingBytes, 0, size, cancellationToken) .ConfigureAwait(false); - if (paddingBytes.Any(b => b != 0)) + for (var i = 0; i < size; i++) { - throw new InvalidFormatException("Padding bytes were non-null"); + if (paddingBytes[i] != 0) + { + throw new InvalidFormatException("Padding bytes were non-null"); + } } } finally @@ -87,7 +91,7 @@ public sealed partial class XZBlock await BaseStream .ReadExactAsync(crc, 0, _checkSize, cancellationToken) .ConfigureAwait(false); - VerifyCheck(crc); + VerifyCheck(crc.AsSpan().Slice(0, _checkSize)); _crcChecked = true; } finally diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs index 4221e3c4..bfb5f3b0 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs @@ -98,10 +98,17 @@ public sealed partial class XZBlock : XZReadOnlyStream private void CheckCrc() { - var crc = new byte[_checkSize]; - BaseStream.ReadExact(crc, 0, _checkSize); - VerifyCheck(crc); - _crcChecked = true; + var crc = ArrayPool.Shared.Rent(_checkSize); + try + { + BaseStream.ReadExact(crc, 0, _checkSize); + VerifyCheck(crc.AsSpan().Slice(0, _checkSize)); + _crcChecked = true; + } + finally + { + ArrayPool.Shared.Return(crc); + } } private void UpdateCheck(byte[] buffer, int offset, int count) @@ -127,7 +134,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } } - private void VerifyCheck(byte[] expected) + private void VerifyCheck(ReadOnlySpan expected) { switch (_checkType) { @@ -147,7 +154,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } } - private static void GetLittleEndianBytes(uint value, byte[] expected) + private static void GetLittleEndianBytes(uint value, ReadOnlySpan expected) { var bytes = ArrayPool.Shared.Rent(sizeof(uint)); try @@ -164,7 +171,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } } - private static void GetLittleEndianBytes(ulong value, byte[] expected) + private static void GetLittleEndianBytes(ulong value, ReadOnlySpan expected) { var bytes = ArrayPool.Shared.Rent(sizeof(ulong)); try @@ -181,7 +188,7 @@ public sealed partial class XZBlock : XZReadOnlyStream } } - private void FinalizeSha256Check(byte[] expected) + private void FinalizeSha256Check(ReadOnlySpan expected) { _sha256.NotNull().TransformFinalBlock(Array.Empty(), 0, 0); if (!expected.SequenceEqual(_sha256.NotNull().Hash)) diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index 5dd0348f..5a32b4ee 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -17,7 +17,7 @@ public class LzmaStreamAsyncTests : TestBase { using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "bad-1-lzma2-7.xz")); - var xz = new XZStream(stream); + using var xz = new XZStream(stream); await Assert.ThrowsAsync(async () => await xz.TransferToAsync(Stream.Null, long.MaxValue) ); From 2d041e5c762c16eb92303707f3b6357cb4a7c2f9 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 14:31:36 +0100 Subject: [PATCH 04/12] Fixed XZ block CRC64 computation and added notes on XZ format specifics --- docs/FORMATS.md | 8 +++++++- src/SharpCompress/Compressors/Xz/Crc64.cs | 12 ++++++++++++ src/SharpCompress/Compressors/Xz/XZBlock.cs | 5 ++--- tests/SharpCompress.Test/Xz/Crc64Tests.cs | 8 ++++++++ tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs | 6 +++--- tests/SharpCompress.Test/Xz/XZBlockTests.cs | 6 +++--- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/FORMATS.md b/docs/FORMATS.md index 5b149b48..3504d659 100644 --- a/docs/FORMATS.md +++ b/docs/FORMATS.md @@ -48,9 +48,15 @@ 2. Skip through the decompressed data to reach each file's starting position **Recommendation**: For best performance with 7Zip archives, use synchronous extraction methods (`MoveToNextEntry()` and `WriteEntryToDirectory()`) when possible. Use async methods only when you need to avoid blocking the thread (e.g., in UI applications or async-only contexts). - + **Technical Details**: 7Zip archives group files into "folders" (compression units), where all files in a folder share one continuous LZMA-compressed stream. The LZMA decoder maintains internal state (dictionary window, decoder positions) that assumes sequential, non-interruptible processing. Async operations can yield control during awaits, which would corrupt this shared state. To avoid this, async extraction creates a fresh decoder stream for each file. +### XZ Format Notes + +- XZ is a container format around LZMA2-compressed blocks, not just raw LZMA/LZMA2 data. +- XZ streams can include per-block integrity checks selected by the stream header: CRC32, CRC64/XZ, SHA-256, or none. SharpCompress validates these checks while reading XZ blocks. +- Raw LZMA/LZMA2 decoding does not provide the same container-level CRC validation; it only validates what the decoder format itself can detect, such as malformed compressed data or invalid end markers. + ## Compression Streams For those who want to directly compress/decompress bits. The single file formats are represented here as well. However, BZip2, LZip and XZ have no metadata (GZip has a little) so using them without something like a Tar file makes little sense. diff --git a/src/SharpCompress/Compressors/Xz/Crc64.cs b/src/SharpCompress/Compressors/Xz/Crc64.cs index e37d04a9..3cb7fbe1 100644 --- a/src/SharpCompress/Compressors/Xz/Crc64.cs +++ b/src/SharpCompress/Compressors/Xz/Crc64.cs @@ -6,10 +6,13 @@ namespace SharpCompress.Compressors.Xz; public static class Crc64 { public const ulong DefaultSeed = 0x0; + internal const ulong XZ_SEED = 0xffffffffffffffff; internal static ulong[]? Table; + private static ulong[]? _xzTable; public const ulong Iso3309Polynomial = 0xD800000000000000; + private const ulong XZ_POLYNOMIAL = 0xC96C5795D7870F42; public static ulong Compute(byte[] buffer) => Compute(DefaultSeed, buffer); @@ -20,6 +23,15 @@ public static class Crc64 return CalculateHash(seed, Table, buffer); } + public static ulong ComputeXz(byte[] buffer) => ~UpdateXz(XZ_SEED, buffer); + + public static ulong UpdateXz(ulong seed, ReadOnlySpan buffer) + { + _xzTable ??= CreateTable(XZ_POLYNOMIAL); + + return CalculateHash(seed, _xzTable, buffer); + } + public static ulong CalculateHash(ulong seed, ulong[] table, ReadOnlySpan buffer) { var crc = seed; diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs index bfb5f3b0..81db843b 100644 --- a/src/SharpCompress/Compressors/Xz/XZBlock.cs +++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs @@ -21,7 +21,7 @@ public sealed partial class XZBlock : XZReadOnlyStream private readonly CheckType _checkType; private readonly int _checkSize; private uint _crc32 = Crc32.DefaultSeed; - private ulong _crc64 = Crc64.DefaultSeed; + private ulong _crc64 = Crc64.XZ_SEED; private readonly SHA256? _sha256; private bool _streamConnected; private int _numFilters; @@ -125,8 +125,7 @@ public sealed partial class XZBlock : XZReadOnlyStream _crc32 = Crc32.Update(_crc32, bytes); break; case CheckType.CRC64: - Crc64.Table ??= Crc64.CreateTable(Crc64.Iso3309Polynomial); - _crc64 = Crc64.CalculateHash(_crc64, Crc64.Table, bytes); + _crc64 = Crc64.UpdateXz(_crc64, bytes); break; case CheckType.SHA256: _sha256.NotNull().TransformBlock(buffer, offset, count, null, 0); diff --git a/tests/SharpCompress.Test/Xz/Crc64Tests.cs b/tests/SharpCompress.Test/Xz/Crc64Tests.cs index 8cf71e1e..e2338c64 100644 --- a/tests/SharpCompress.Test/Xz/Crc64Tests.cs +++ b/tests/SharpCompress.Test/Xz/Crc64Tests.cs @@ -27,4 +27,12 @@ public class Crc64Tests Assert.Equal((ulong)0x416B4150508661EE, actual); } + + [Fact] + public void XzCheckString() + { + var actual = Crc64.ComputeXz(Encoding.ASCII.GetBytes("123456789")); + + Assert.Equal(0x995DC9BBDF1939FAUL, actual); + } } diff --git a/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs b/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs index 6460ed53..9d59d6d4 100644 --- a/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs +++ b/tests/SharpCompress.Test/Xz/XZBlockAsyncTests.cs @@ -85,7 +85,7 @@ public class XzBlockAsyncTests : XzTestsBase { using var xzBlock = new XZBlock(CompressedStream, CheckType.CRC64, 8); using var sr = new StreamReader(xzBlock); - Assert.Equal(await sr.ReadToEndAsync().ConfigureAwait(false), Original); + Assert.Equal(Original, await sr.ReadToEndAsync().ConfigureAwait(false)); } [Fact] @@ -101,8 +101,8 @@ public class XzBlockAsyncTests : XzTestsBase [Fact] public async ValueTask SkipsPaddingWhenPresentAsync() { - // CompressedIndexedStream's first block has 1-byte padding. - using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC64, 8); + // CompressedIndexedStream uses CRC32 checks. + using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC32, 4); using var sr = new StreamReader(xzBlock); await sr.ReadToEndAsync().ConfigureAwait(false); Assert.Equal(0L, CompressedIndexedStream.Position % 4L); diff --git a/tests/SharpCompress.Test/Xz/XZBlockTests.cs b/tests/SharpCompress.Test/Xz/XZBlockTests.cs index 3b06008e..9b6efe6e 100644 --- a/tests/SharpCompress.Test/Xz/XZBlockTests.cs +++ b/tests/SharpCompress.Test/Xz/XZBlockTests.cs @@ -72,7 +72,7 @@ public class XzBlockTests : XzTestsBase { var xzBlock = new XZBlock(CompressedStream, CheckType.CRC64, 8); using var sr = new StreamReader(xzBlock); - Assert.Equal(sr.ReadToEnd(), Original); + Assert.Equal(Original, sr.ReadToEnd()); } [Fact] @@ -88,8 +88,8 @@ public class XzBlockTests : XzTestsBase [Fact] public void SkipsPaddingWhenPresent() { - // CompressedIndexedStream's first block has 1-byte padding. - using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC64, 8); + // CompressedIndexedStream uses CRC32 checks. + using var xzBlock = new XZBlock(CompressedIndexedStream, CheckType.CRC32, 4); using var sr = new StreamReader(xzBlock); sr.ReadToEnd(); Assert.Equal(0L, CompressedIndexedStream.Position % 4L); From 78a22189825f1076315f0b7e96ddc12ab2c0e85d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 15:09:13 +0100 Subject: [PATCH 05/12] Handle lzma end marker correctly --- .../Compressors/LZMA/LzmaDecoder.cs | 2 ++ .../Compressors/LZMA/LzmaStream.Async.cs | 21 ++++++++--- .../Compressors/LZMA/LzmaStream.cs | 36 ++++++++++++++++--- .../Streams/LzmaStreamAsyncTests.cs | 2 +- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index bdc189ff..02fd16d7 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -11,6 +11,8 @@ namespace SharpCompress.Compressors.LZMA; public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { + internal bool HasEndMarker => _rep0 == uint.MaxValue; + public void Dispose() { _outWindow?.Dispose(); diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 4ad30127..c89fb923 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -92,6 +92,11 @@ public partial class LzmaStream if (control == 0x00) { + if (_isLzma2 && _decoder is { HasEndMarker: true }) + { + throw new DataErrorException(); + } + _endReached = true; return; } @@ -213,10 +218,9 @@ public partial class LzmaStream await _decoder! .CodeAsync(_dictionarySize, _outWindow, _rangeDecoder, cancellationToken) .ConfigureAwait(false) - && _outputSize < 0 ) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -227,6 +231,11 @@ public partial class LzmaStream if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + if ( !_rangeDecoder.IsFinished || (_rangeDecoderLimit >= 0 && _rangeDecoder._total != _rangeDecoderLimit) @@ -325,10 +334,9 @@ public partial class LzmaStream await _decoder! .CodeAsync(_dictionarySize, _outWindow, _rangeDecoder, cancellationToken) .ConfigureAwait(false) - && _outputSize < 0 ) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -339,6 +347,11 @@ public partial class LzmaStream if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + if ( !_rangeDecoder.IsFinished || (_rangeDecoderLimit >= 0 && _rangeDecoder._total != _rangeDecoderLimit) diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 38bb2eca..7e2408ab 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -277,9 +277,9 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { _inputPosition += _outWindow.CopyStream(_inputStream, toProcess); } - else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) + else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder)) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var read = _outWindow.Read(buffer, offset, toProcess); @@ -290,6 +290,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + // Check range corruption scenario if ( !_rangeDecoder.IsFinished @@ -368,9 +373,9 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { _inputPosition += _outWindow.CopyStream(_inputStream, 1); } - else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) + else if (_decoder!.Code(_dictionarySize, _outWindow, _rangeDecoder)) { - _availableBytes = _outWindow.AvailableBytes; + HandleEndMarker(); } var value = _outWindow.ReadByte(); @@ -379,6 +384,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (_availableBytes == 0 && !_uncompressedChunk) { + if (_isLzma2 && _decoder!.HasEndMarker) + { + throw new DataErrorException(); + } + // Check range corruption scenario if ( !_rangeDecoder.IsFinished @@ -413,6 +423,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable if (control == 0x00) { + if (_isLzma2 && _decoder is { HasEndMarker: true }) + { + throw new DataErrorException(); + } + _endReached = true; return; } @@ -472,6 +487,19 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } } + private void HandleEndMarker() + { + if (_isLzma2) + { + throw new DataErrorException(); + } + + if (_outputSize < 0) + { + _availableBytes = _outWindow.AvailableBytes; + } + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index 5a32b4ee..cc41e831 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -18,7 +18,7 @@ public class LzmaStreamAsyncTests : TestBase using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "bad-1-lzma2-7.xz")); using var xz = new XZStream(stream); - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAnyAsync(async () => await xz.TransferToAsync(Stream.Null, long.MaxValue) ); } From c36a916c842cfdf334289e646a4b3c3a3d2a02cb Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Jun 2026 15:11:52 +0100 Subject: [PATCH 06/12] add XZ LZMA skill --- .agents/skills/xz-lzma-format/SKILL.md | 22 +++ .../references/xz-lzma-format.md | 174 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 .agents/skills/xz-lzma-format/SKILL.md create mode 100644 .agents/skills/xz-lzma-format/references/xz-lzma-format.md diff --git a/.agents/skills/xz-lzma-format/SKILL.md b/.agents/skills/xz-lzma-format/SKILL.md new file mode 100644 index 00000000..7defd7e9 --- /dev/null +++ b/.agents/skills/xz-lzma-format/SKILL.md @@ -0,0 +1,22 @@ +--- +name: xz-lzma-format +description: Reference the XZ container format and LZMA/LZMA2 decoder behavior. Use when an AI agent needs to answer questions or make code changes involving XZ headers, blocks, indexes, checks, CRC64/XZ, LZMA2 chunks, LZMA end markers, corrupt .xz test files, or SharpCompress XZ/LZMA parsing and decompression behavior. +--- + +# XZ and LZMA Format + +Use this skill for work at the boundary between the XZ container and the LZMA/LZMA2 compression streams. It captures the key details needed for SharpCompress XZ block parsing, XZ integrity checks, and LZMA2 decoder corruption handling. + +## Reference + +- Read [references/xz-lzma-format.md](references/xz-lzma-format.md) when the task depends on XZ binary layout, XZ block checks, CRC64/XZ parameters, LZMA2 chunk control bytes, LZMA end-of-payload markers, or XZ Utils bad-file expectations. +- Treat XZ as a container around filter chains. Do not assume raw LZMA/LZMA2 behavior is equivalent to XZ stream validation. +- Use the linked XZ Utils/liblzma sources in the reference when matching corruption behavior. The test corpus contains intentionally bad files that must throw even when they can produce all expected output bytes. + +## Workflow + +1. Identify the layer involved: XZ stream header/footer, block header, compressed data padding, block check, index, filter chain, LZMA2 chunk framing, or raw LZMA range decoding. +2. Open `references/xz-lzma-format.md` and cross-check the relevant spec/source section before changing parser or decoder code. +3. For XZ checksum work, verify the stream check type from the XZ header. Use CRC32, CRC64/XZ, SHA-256, or no check according to the header, not according to a test fixture assumption. +4. For LZMA2 corruption work, compare SharpCompress behavior against the XZ Utils test corpus notes and liblzma decoder state model. +5. Test both sync and async paths. Relevant files are `XZBlock.cs`, `XZBlock.Async.cs`, `XZStream.cs`, `XZStream.Async.cs`, `LzmaStream.cs`, `LzmaStream.Async.cs`, `LzmaDecoder.cs`, and `LzmaDecoder.Async.cs`. diff --git a/.agents/skills/xz-lzma-format/references/xz-lzma-format.md b/.agents/skills/xz-lzma-format/references/xz-lzma-format.md new file mode 100644 index 00000000..60538f6f --- /dev/null +++ b/.agents/skills/xz-lzma-format/references/xz-lzma-format.md @@ -0,0 +1,174 @@ +# XZ and LZMA/LZMA2 Reference + +This reference summarizes the XZ container and LZMA/LZMA2 decoder facts that matter for SharpCompress maintenance. It is a local guide, not a full copy of the specs. + +## Upstream References + +- XZ file format specification: `https://raw.githubusercontent.com/tukaani-project/xz/master/doc/xz-file-format.txt` +- liblzma LZMA2 decoder: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/lzma/lzma2_decoder.c` +- liblzma LZMA decoder: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/lzma/lzma_decoder.c` +- XZ Utils test file descriptions: `https://raw.githubusercontent.com/tukaani-project/xz/master/tests/files/README` +- XZ range decoder reference, useful for `rc_is_finished` and normalization behavior: `https://raw.githubusercontent.com/tukaani-project/xz/master/src/liblzma/rangecoder/range_decoder.h` + +## SharpCompress Pointers + +- XZ container stream: `src/SharpCompress/Compressors/Xz/XZStream.cs`, `src/SharpCompress/Compressors/Xz/XZStream.Async.cs` +- XZ block parsing/checks: `src/SharpCompress/Compressors/Xz/XZBlock.cs`, `src/SharpCompress/Compressors/Xz/XZBlock.Async.cs` +- XZ header/footer/index: `XZHeader.cs`, `XZFooter.cs`, `XZIndex.cs`, `XZIndexRecord.cs`, and async counterparts. +- XZ LZMA2 filter wrapper: `src/SharpCompress/Compressors/Xz/Filters/Lzma2Filter.cs`, `Lzma2Filter.Async.cs` +- LZMA/LZMA2 stream and decoder: `src/SharpCompress/Compressors/LZMA/LzmaStream.cs`, `LzmaStream.Async.cs`, `LzmaDecoder.cs`, `LzmaDecoder.Async.cs`, `RangeCoder/RangeCoder.cs`, `RangeCoder/RangeCoder.Async.cs` +- Core tests: `tests/SharpCompress.Test/Xz/*`, `tests/SharpCompress.Test/Streams/LzmaStreamTests.cs`, `tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs` +- Corruption fixture discussed here: `tests/TestArchives/Archives/bad-1-lzma2-7.xz` + +## XZ Container Structure + +An XZ file is one or more XZ streams. A typical single stream is: + +```text +Stream Header -> Block(s) -> Index -> Stream Footer +``` + +Important layout rules: + +- XZ files and XZ streams are aligned to four-byte boundaries. +- Stream header magic is `FD 37 7A 58 5A 00`. +- Stream footer magic is `59 5A` (`YZ`). +- Stream header/footer flags include the check type used for every block in the stream. +- A block consists of `Block Header`, `Compressed Data`, `Block Padding`, and `Check`. +- Block padding is 0-3 null bytes and makes the block size a multiple of four. +- The Index contains one record per block: `Unpadded Size` and `Uncompressed Size`. + +## Variable-Length Integers + +XZ variable-length integers encode seven data bits per byte. The high bit means continuation. Current XZ limits the encoded integer to nine bytes/63 bits. + +SharpCompress implementation: + +- `MultiByteIntegers.ReadXZInteger` reads these values. +- It rejects overlong encodings when a continuation byte is `0x00`. +- Use this for block sizes, filter IDs, filter property sizes, index counts, and index record fields. + +## XZ Checks Versus Raw LZMA + +XZ checks are container-level integrity checks over uncompressed block data. Raw LZMA/LZMA2 decoding does not provide the same container-level CRC validation. + +Supported XZ check IDs in SharpCompress: + +- `0x00`: none, 0 bytes. +- `0x01`: CRC32, 4 bytes. +- `0x04`: CRC64/XZ, 8 bytes. +- `0x0A`: SHA-256, 32 bytes. + +CRC64/XZ details: + +- Polynomial: reflected ECMA polynomial `0xC96C5795D7870F42`. +- Initial value: `0xffffffffffffffff`. +- Final XOR: `0xffffffffffffffff`. +- Stored little-endian in the block check field. +- Test vector for `"123456789"`: `0x995DC9BBDF1939FA`. + +Common pitfall: + +- CRC64/XZ is not the older `Iso3309Polynomial = 0xD800000000000000` path that existed in SharpCompress's generic `Crc64` helper. Using that produces wrong XZ block check values. + +## XZ Block Check Handling + +When reading an `XZBlock`: + +1. Parse and CRC-validate the block header. +2. Build the filter chain in reverse order from the List of Filter Flags. +3. Read uncompressed bytes through the filter chain and update the selected check over the uncompressed bytes. +4. At block end, skip/validate block padding. +5. Read the check field and compare to the computed value. + +Important behavior: + +- A short `Stream.Read` result does not universally mean EOF. Be careful when using `bytesRead != count` as an end-of-block signal. +- Tests using `StreamReader.ReadToEnd()` often expose end-of-block behavior because they force padding/check validation. +- The check type must match the XZ stream header. For example, a fixture may have one XZ stream using CRC32 and another using CRC64; do not hard-code CRC64 in block tests. + +## LZMA2 Chunks + +LZMA2 is the only LZMA-family filter defined for XZ (`Filter ID 0x21`). Raw LZMA is not an XZ filter. + +LZMA2 control byte categories from liblzma: + +- `0x00`: LZMA2 end marker. +- `0x01` or `>= 0xE0`: dictionary reset; the next LZMA chunk must set new properties. +- `>= 0x80`: LZMA chunk. The control byte and following two bytes encode uncompressed chunk size. The next two bytes encode compressed chunk size. Some control values also provide new LZMA properties. +- `0x02`: uncompressed chunk without dictionary reset. +- `0x01`: uncompressed chunk with dictionary reset. +- `0x03..0x7F`: invalid/reserved control values. + +For LZMA chunks, the LZMA2 decoder must track: + +- Exact uncompressed chunk size. +- Exact compressed chunk size. +- Whether LZMA properties are needed. +- Whether dictionary reset is required. +- Whether the inner LZMA stream saw an LZMA end-of-payload marker. + +## LZMA End-Of-Payload Marker In LZMA2 + +The XZ Utils bad-file corpus describes `bad-1-lzma2-7.xz` as: + +```text +bad-1-lzma2-7.xz has EOPM at LZMA level. +``` + +Meaning: + +- The outer XZ container can be parsed. +- The LZMA2 stream can produce all advertised uncompressed bytes. +- The inner raw LZMA decoder still reaches an LZMA end-of-payload marker (`rep0 == uint.MaxValue`). +- LZMA2 must reject this. End-of-payload markers are for raw LZMA cases with unknown size; they are not valid as an LZMA-level terminator inside an LZMA2 chunk. + +liblzma behavior: + +- `lzma2_decoder.c` calls the inner LZMA decoder with a known `uncompressed_size` and `allow_eopm = false`. +- `lzma_decoder.c` treats EOPM as data error when EOPM is not valid. +- The XZ Utils test README says all `bad-*` files must cause decoder errors. + +SharpCompress maintenance guidance: + +- If a bad LZMA2 fixture produces all output bytes but `xz --test` reports corrupt data, inspect the inner decoder state, not just output length or XZ block check. +- In SharpCompress, `Decoder.HasEndMarker => _rep0 == uint.MaxValue` is the useful signal for the `bad-1-lzma2-7.xz` case. +- Validate both sync and async paths; `Stream.CopyToAsync` can use byte-array or `Memory` read overloads depending on target framework and wrapper stream. + +## Exception Expectations + +`DataErrorException` is internal and derives from `SharpCompressException`. Some public XZ parsing failures throw `InvalidFormatException`; raw decoder corruption may surface as `SharpCompressException` via `DataErrorException` unless the wrapper maps it. + +Testing guidance: + +- Use exact `InvalidFormatException` when the code path is XZ header/footer/block/index/check validation. +- Use `Assert.ThrowsAnyAsync` or equivalent when the desired behavior is simply that corrupt LZMA/LZMA2 data is rejected. +- Do not weaken tests to accept no exception for XZ Utils `bad-*` fixtures. + +## Useful Commands + +Use the system `xz` tool as an oracle when available: + +```bash +xz --test --verbose tests/TestArchives/Archives/bad-1-lzma2-7.xz +xz --robot --list --verbose tests/TestArchives/Archives/bad-1-lzma2-7.xz +``` + +Expected for `bad-1-lzma2-7.xz`: + +```text +xz: tests/TestArchives/Archives/bad-1-lzma2-7.xz: Compressed data is corrupt +``` + +Targeted SharpCompress tests: + +```bash +dotnet test tests/SharpCompress.Test/SharpCompress.Test.csproj --framework net10.0 --filter "FullyQualifiedName~SharpCompress.Test.Streams.LzmaStream|FullyQualifiedName~SharpCompress.Test.Xz" +``` + +## Current SharpCompress Gotchas + +- XZ index CRC32 verification may still be incomplete; check `XZIndex.VerifyCrc32` before relying on index corruption detection. +- XZ block/index size semantics are easy to confuse. `Unpadded Size` excludes block padding but includes header, compressed data, and check. `Uncompressed Size` is raw output size. +- LZMA2 chunks include their own compressed and uncompressed chunk sizes; these are separate from XZ block header/index sizes. +- `StreamReader.ReadToEnd()` and `TransferToAsync(Stream.Null, long.MaxValue)` are useful for forcing full-stream validation. From 2866bfbd540a794a72d780cb16273de299b09662 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 15 Jun 2026 09:17:05 +0100 Subject: [PATCH 07/12] First pass with zip --- docs/API.md | 11 +- docs/USAGE.md | 3 +- .../Archives/IArchiveEntryExtensions.cs | 31 ++- .../Common/ChecksumDescriptor.cs | 12 ++ .../Common/ChecksumValidationStream.cs | 140 +++++++++++++ src/SharpCompress/Common/Entry.cs | 2 + src/SharpCompress/Common/ExtractionOptions.cs | 9 + src/SharpCompress/Common/IEntryExtensions.cs | 18 ++ .../Zip/Headers/DirectoryEntryHeader.Async.cs | 1 + .../Zip/Headers/DirectoryEntryHeader.cs | 1 + .../Zip/Headers/LocalEntryHeader.Async.cs | 1 + .../Common/Zip/Headers/LocalEntryHeader.cs | 1 + .../Common/Zip/Headers/ZipFileEntry.cs | 2 + .../Common/Zip/SeekableZipHeaderFactory.cs | 1 + .../Zip/StreamingZipHeaderFactory.Async.cs | 3 + .../Common/Zip/StreamingZipHeaderFactory.cs | 3 + src/SharpCompress/Common/Zip/ZipEntry.cs | 43 ++++ .../Common/Zip/ZipHeaderFactory.Async.cs | 1 + .../Common/Zip/ZipHeaderFactory.cs | 1 + src/SharpCompress/Crypto/Crc32Stream.cs | 6 +- .../Readers/IAsyncReaderExtensions.cs | 27 ++- .../Readers/IReaderExtensions.cs | 25 ++- .../OptionsUsabilityTests.cs | 1 + .../Zip/ZipCrcExtractionTests.cs | 196 ++++++++++++++++++ 24 files changed, 509 insertions(+), 30 deletions(-) create mode 100644 src/SharpCompress/Common/ChecksumDescriptor.cs create mode 100644 src/SharpCompress/Common/ChecksumValidationStream.cs create mode 100644 tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs diff --git a/docs/API.md b/docs/API.md index 19edd6ed..79745918 100644 --- a/docs/API.md +++ b/docs/API.md @@ -320,7 +320,11 @@ var flatOptions = ExtractionOptions.FlatExtract; // No directory structure var metadataOptions = ExtractionOptions.PreserveMetadata; // Keep timestamps and attributes // Tune extraction copy buffering -var extractionOptions = new ExtractionOptions { BufferSize = 131072 }; +var extractionOptions = new ExtractionOptions +{ + BufferSize = 131072, + CheckCrc = true, // Default: validate entry checksums when archive metadata provides one +}; // Factory defaults: // - file path / FileInfo overloads use LeaveStreamOpen = false @@ -429,7 +433,8 @@ var options = new ExtractionOptions { ExtractFullPath = true, // Recreate directory structure Overwrite = true, // Overwrite existing files - PreserveFileTime = true // Keep original timestamps + PreserveFileTime = true, // Keep original timestamps + CheckCrc = true // Validate payload checksums when available }; using (var archive = ZipArchive.OpenArchive("file.zip")) @@ -438,6 +443,8 @@ using (var archive = ZipArchive.OpenArchive("file.zip")) } ``` +`CheckCrc` validates archive-level payload checksums when the format stores reliable metadata, such as ZIP CRC32 values. Formats without payload checksums skip this validation. Decompressor integrity checks that are required to decode a stream may still fail even when `CheckCrc` is disabled. + ### Options matrix ```text diff --git a/docs/USAGE.md b/docs/USAGE.md index 72356614..9666d43f 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -105,6 +105,7 @@ using (var archive = RarArchive.OpenArchive("Test.rar", ReaderOptions.ForFilePat ExtractFullPath = true, Overwrite = true, BufferSize = 131072, + CheckCrc = true, // Default: validate payload checksums when available } ); } @@ -141,7 +142,7 @@ using (var archive = RarArchive.OpenArchive("archive.rar", { archive.WriteToDirectory( @"D:\output", - new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + new ExtractionOptions { ExtractFullPath = true, Overwrite = true, CheckCrc = true } ); } ``` diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 0de4b21c..a9c3a054 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -18,11 +18,12 @@ public static class IArchiveEntryExtensions /// The stream to write the entry content to. /// Optional progress reporter for tracking extraction progress. public void WriteTo(Stream streamToWriteTo, IProgress? progress = null) => - archiveEntry.WriteTo(streamToWriteTo, null, progress); + archiveEntry.WriteTo(streamToWriteTo, null, progress: progress); private void WriteTo( Stream streamToWriteTo, int? bufferSize, + ExtractionOptions? options = null, IProgress? progress = null ) { @@ -32,7 +33,10 @@ public static class IArchiveEntryExtensions } using var entryStream = archiveEntry.OpenEntryStream(); - var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); + var checkedStream = options is null + ? entryStream + : IEntryExtensions.WrapWithChecksumValidation(archiveEntry, entryStream, options); + var sourceStream = WrapWithProgress(checkedStream, archiveEntry, progress); sourceStream.CopyTo(streamToWriteTo, bufferSize ?? Constants.BufferSize); } @@ -49,13 +53,19 @@ public static class IArchiveEntryExtensions ) { await archiveEntry - .WriteToAsync(streamToWriteTo, Constants.BufferSize, progress, cancellationToken) + .WriteToAsync( + streamToWriteTo, + Constants.BufferSize, + progress: progress, + cancellationToken: cancellationToken + ) .ConfigureAwait(false); } private async ValueTask WriteToAsync( Stream streamToWriteTo, int? bufferSize, + ExtractionOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default ) @@ -74,7 +84,10 @@ public static class IArchiveEntryExtensions .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); #endif - var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); + var checkedStream = options is null + ? entryStream + : IEntryExtensions.WrapWithChecksumValidation(archiveEntry, entryStream, options); + var sourceStream = WrapWithProgress(checkedStream, archiveEntry, progress); await sourceStream .CopyToAsync(streamToWriteTo, bufferSize ?? Constants.BufferSize, cancellationToken) .ConfigureAwait(false); @@ -154,13 +167,14 @@ public static class IArchiveEntryExtensions /// public void WriteToFile(string destinationFileName, ExtractionOptions? options = null) { + options ??= new ExtractionOptions(); entry.WriteEntryToFile( destinationFileName, options, (x, fm) => { using var fs = File.Open(x, fm); - entry.WriteTo(fs, options?.BufferSize ?? Constants.BufferSize); + entry.WriteTo(fs, options?.BufferSize ?? Constants.BufferSize, options, null); } ); } @@ -172,7 +186,9 @@ public static class IArchiveEntryExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await entry .WriteEntryToFileAsync( destinationFileName, @@ -181,11 +197,12 @@ public static class IArchiveEntryExtensions { using var fs = File.Open(x, fm); await entry - .WriteToAsync(fs, options?.BufferSize, null, ct) + .WriteToAsync(fs, options.BufferSize, options, null, ct) .ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } } } diff --git a/src/SharpCompress/Common/ChecksumDescriptor.cs b/src/SharpCompress/Common/ChecksumDescriptor.cs new file mode 100644 index 00000000..2dbb6a45 --- /dev/null +++ b/src/SharpCompress/Common/ChecksumDescriptor.cs @@ -0,0 +1,12 @@ +namespace SharpCompress.Common; + +internal enum ChecksumKind +{ + Crc32, +} + +internal readonly record struct ChecksumDescriptor( + ChecksumKind Kind, + long ExpectedValue, + bool IsAvailable +); diff --git a/src/SharpCompress/Common/ChecksumValidationStream.cs b/src/SharpCompress/Common/ChecksumValidationStream.cs new file mode 100644 index 00000000..fdf4ba42 --- /dev/null +++ b/src/SharpCompress/Common/ChecksumValidationStream.cs @@ -0,0 +1,140 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Crypto; + +namespace SharpCompress.Common; + +internal sealed class ChecksumValidationStream : Stream +{ + private readonly Stream _stream; + private readonly ChecksumDescriptor _checksum; + private readonly string _entryName; + private readonly uint[] _crc32Table; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private bool _validated; + + internal ChecksumValidationStream(Stream stream, ChecksumDescriptor checksum, string? entryName) + { + _stream = stream; + _checksum = checksum; + _entryName = string.IsNullOrEmpty(entryName) ? "Entry" : entryName!; + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _stream.Length; + + public override long Position + { + get => _stream.Position; + set => throw new NotSupportedException(); + } + + public override void Flush() => _stream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _stream.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } +#endif + + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value == -1) + { + Validate(); + } + else + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, (byte)value); + } + + return value; + } + + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (read > 0) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + return; + } + + Validate(); + } + + private void Validate() + { + if (_validated) + { + return; + } + + _validated = true; + + if (_checksum.Kind != ChecksumKind.Crc32) + { + return; + } + + var actual = ~_seed; + var expected = unchecked((uint)_checksum.ExpectedValue); + if (actual != expected) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expected:X8}, actual 0x{actual:X8}." + ); + } + } +} diff --git a/src/SharpCompress/Common/Entry.cs b/src/SharpCompress/Common/Entry.cs index 1942ba46..6c724c32 100644 --- a/src/SharpCompress/Common/Entry.cs +++ b/src/SharpCompress/Common/Entry.cs @@ -84,6 +84,8 @@ public abstract class Entry : IEntry internal virtual void Close() { } + internal virtual ChecksumDescriptor Checksum => default; + /// /// Entry file attribute. /// diff --git a/src/SharpCompress/Common/ExtractionOptions.cs b/src/SharpCompress/Common/ExtractionOptions.cs index b99658f8..04b33416 100644 --- a/src/SharpCompress/Common/ExtractionOptions.cs +++ b/src/SharpCompress/Common/ExtractionOptions.cs @@ -43,6 +43,15 @@ public sealed record ExtractionOptions : IExtractionOptions /// public int BufferSize { get; set; } = Constants.BufferSize; + /// + /// Validate archive entry checksums during extraction when checksum metadata is available. + /// + /// + /// Formats without payload checksums skip this validation. Compression-format integrity + /// checks that are required to decode data may still fail even when this is disabled. + /// + public bool CheckCrc { get; set; } = true; + /// /// Delegate for writing symbolic links to disk. /// The first parameter is the source path (where the symlink is created). diff --git a/src/SharpCompress/Common/IEntryExtensions.cs b/src/SharpCompress/Common/IEntryExtensions.cs index d3158f03..84645910 100644 --- a/src/SharpCompress/Common/IEntryExtensions.cs +++ b/src/SharpCompress/Common/IEntryExtensions.cs @@ -5,6 +5,24 @@ namespace SharpCompress.Common; internal static partial class IEntryExtensions { + internal static Stream WrapWithChecksumValidation( + IEntry entry, + Stream source, + ExtractionOptions? options + ) + { + if (options?.CheckCrc != false && entry is Entry typedEntry) + { + var checksum = typedEntry.Checksum; + if (checksum.IsAvailable) + { + return new ChecksumValidationStream(source, checksum, entry.Key); + } + } + + return source; + } + extension(IEntry entry) { /// diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs index c09cac91..af978f64 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.Async.cs @@ -22,6 +22,7 @@ internal partial class DirectoryEntryHeader .ReadUInt16Async() .ConfigureAwait(false); Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + IsCrcAvailable = true; CompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); UncompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); var nameLength = await reader.ReadUInt16Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs index e4e0f331..f41c6047 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs @@ -17,6 +17,7 @@ internal partial class DirectoryEntryHeader : ZipFileEntry OriginalLastModifiedTime = LastModifiedTime = reader.ReadUInt16(); OriginalLastModifiedDate = LastModifiedDate = reader.ReadUInt16(); Crc = reader.ReadUInt32(); + IsCrcAvailable = true; CompressedSize = reader.ReadUInt32(); UncompressedSize = reader.ReadUInt32(); var nameLength = reader.ReadUInt16(); diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs index 950494df..2e96c942 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.Async.cs @@ -20,6 +20,7 @@ internal partial class LocalEntryHeader .ReadUInt16Async() .ConfigureAwait(false); Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + IsCrcAvailable = !Flags.HasFlag(HeaderFlags.UsePostDataDescriptor); CompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); UncompressedSize = await reader.ReadUInt32Async().ConfigureAwait(false); var nameLength = await reader.ReadUInt16Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs index d9490137..9d4512b1 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs @@ -16,6 +16,7 @@ internal partial class LocalEntryHeader : ZipFileEntry OriginalLastModifiedTime = LastModifiedTime = reader.ReadUInt16(); OriginalLastModifiedDate = LastModifiedDate = reader.ReadUInt16(); Crc = reader.ReadUInt32(); + IsCrcAvailable = !Flags.HasFlag(HeaderFlags.UsePostDataDescriptor); CompressedSize = reader.ReadUInt32(); UncompressedSize = reader.ReadUInt32(); var nameLength = reader.ReadUInt16(); diff --git a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs index b62b6a67..e7f0b042 100644 --- a/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs +++ b/src/SharpCompress/Common/Zip/Headers/ZipFileEntry.cs @@ -80,6 +80,8 @@ internal abstract partial class ZipFileEntry(ZipHeaderType type, IArchiveEncodin internal uint Crc { get; set; } + internal bool IsCrcAvailable { get; set; } + protected void LoadExtra(byte[] extra) { for (var i = 0; i < extra.Length; ) diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs index 22304b9d..7a88a480 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs @@ -159,6 +159,7 @@ internal sealed partial class SeekableZipHeaderFactory : ZipHeaderFactory if (FlagUtility.HasFlag(localEntryHeader.Flags, HeaderFlags.UsePostDataDescriptor)) { localEntryHeader.Crc = directoryEntryHeader.Crc; + localEntryHeader.IsCrcAvailable = true; localEntryHeader.CompressedSize = directoryEntryHeader.CompressedSize; localEntryHeader.UncompressedSize = directoryEntryHeader.UncompressedSize; } diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs index 80ff16b4..4e3bcc3a 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs @@ -124,6 +124,7 @@ internal sealed partial class StreamingZipHeaderFactory .ConfigureAwait(false); } lastEntryHeader.Crc = crc; + lastEntryHeader.IsCrcAvailable = true; //attempt 32bit read ulong compressedSize = await _reader @@ -205,6 +206,7 @@ internal sealed partial class StreamingZipHeaderFactory .ConfigureAwait(false); } lastEntryHeader.Crc = crc; + lastEntryHeader.IsCrcAvailable = true; // The DataDescriptor can be either 64bit or 32bit var compressedSize = await _reader @@ -283,6 +285,7 @@ internal sealed partial class StreamingZipHeaderFactory localHeader.UncompressedSize = directoryHeader.Size; localHeader.CompressedSize = directoryHeader.CompressedSize; localHeader.Crc = (uint)directoryHeader.Crc; + localHeader.IsCrcAvailable = true; } // If we have CompressedSize, there is data to be read diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs index 7dcc4c54..c955afb0 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs @@ -59,6 +59,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory crc = reader.ReadUInt32(); } _lastEntryHeader.Crc = crc; + _lastEntryHeader.IsCrcAvailable = true; //attempt 32bit read ulong compSize = reader.ReadUInt32(); @@ -121,6 +122,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory crc = reader.ReadUInt32(); } _lastEntryHeader.Crc = crc; + _lastEntryHeader.IsCrcAvailable = true; // The DataDescriptor can be either 64bit or 32bit var compressed_size = reader.ReadUInt32(); @@ -188,6 +190,7 @@ internal partial class StreamingZipHeaderFactory : ZipHeaderFactory local_header.UncompressedSize = dir_header.Size; local_header.CompressedSize = dir_header.CompressedSize; local_header.Crc = (uint)dir_header.Crc; + local_header.IsCrcAvailable = true; } // If we have CompressedSize, there is data to be read diff --git a/src/SharpCompress/Common/Zip/ZipEntry.cs b/src/SharpCompress/Common/Zip/ZipEntry.cs index 753c6dc7..bfe1a50d 100644 --- a/src/SharpCompress/Common/Zip/ZipEntry.cs +++ b/src/SharpCompress/Common/Zip/ZipEntry.cs @@ -89,6 +89,49 @@ public class ZipEntry : Entry public override long Crc => _filePart?.Header.Crc ?? 0; + internal override ChecksumDescriptor Checksum + { + get + { + if ( + _filePart is null + || IsDirectory + || !_filePart.Header.IsCrcAvailable + || !IsReliableCrcMetadata(_filePart.Header) + ) + { + return default; + } + + return new ChecksumDescriptor( + ChecksumKind.Crc32, + _filePart.Header.Crc, + IsAvailable: true + ); + } + } + + private static bool IsReliableCrcMetadata(ZipFileEntry header) + { + if (header.CompressionMethod != ZipCompressionMethod.WinzipAes) + { + return true; + } + + var aesExtraData = header.Extra.FirstOrDefault(x => x.Type == ExtraDataType.WinZipAes); + if (aesExtraData is null || aesExtraData.DataBytes.Length < MinimumWinZipAesExtraDataLength) + { + return false; + } + + var vendorVersion = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian( + aesExtraData.DataBytes + ); + + // WinZip AES AE-2 stores a zero CRC field by design and relies on AES authentication. + return vendorVersion == 0x0001; + } + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs index c31e24a6..bc5944b4 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.Async.cs @@ -44,6 +44,7 @@ internal partial class ZipHeaderFactory ) { _lastEntryHeader.Crc = await reader.ReadUInt32Async().ConfigureAwait(false); + _lastEntryHeader.IsCrcAvailable = true; _lastEntryHeader.CompressedSize = zip64 ? (long)await reader.ReadUInt64Async().ConfigureAwait(false) : await reader.ReadUInt32Async().ConfigureAwait(false); diff --git a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs index 8d0ffdb4..e1ebb8c1 100644 --- a/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/ZipHeaderFactory.cs @@ -66,6 +66,7 @@ internal partial class ZipHeaderFactory ) { _lastEntryHeader.Crc = reader.ReadUInt32(); + _lastEntryHeader.IsCrcAvailable = true; _lastEntryHeader.CompressedSize = zip64 ? (long)reader.ReadUInt64() : reader.ReadUInt32(); diff --git a/src/SharpCompress/Crypto/Crc32Stream.cs b/src/SharpCompress/Crypto/Crc32Stream.cs index b3294831..67dcb913 100644 --- a/src/SharpCompress/Crypto/Crc32Stream.cs +++ b/src/SharpCompress/Crypto/Crc32Stream.cs @@ -119,7 +119,7 @@ public sealed class Crc32Stream : Stream public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer) => ~CalculateCrc(InitializeTable(polynomial), seed, buffer); - private static uint[] InitializeTable(uint polynomial) + internal static uint[] InitializeTable(uint polynomial) { if (polynomial == DEFAULT_POLYNOMIAL && _defaultTable != null) { @@ -153,7 +153,7 @@ public sealed class Crc32Stream : Stream return createTable; } - private static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan buffer) + internal static uint CalculateCrc(uint[] table, uint crc, ReadOnlySpan buffer) { unchecked { @@ -165,6 +165,6 @@ public sealed class Crc32Stream : Stream return crc; } - private static uint CalculateCrc(uint[] table, uint crc, byte b) => + internal static uint CalculateCrc(uint[] table, uint crc, byte b) => (crc >> 8) ^ table[(crc ^ b) & 0xFF]; } diff --git a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs index a982982c..7515b257 100644 --- a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs +++ b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs @@ -36,7 +36,9 @@ public static class IAsyncReaderExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await reader .Entry.WriteEntryToFileAsync( destinationFileName, @@ -44,12 +46,12 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) - .ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options, ct).ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } /// /// Extract all remaining unread entries to specific directory asynchronously, retaining filename @@ -72,7 +74,9 @@ public static class IAsyncReaderExtensions string destinationFileName, ExtractionOptions? options = null, CancellationToken cancellationToken = default - ) => + ) + { + options ??= new ExtractionOptions(); await reader .Entry.WriteEntryToFileAsync( destinationFileName, @@ -80,12 +84,12 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) - .ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options, ct).ConfigureAwait(false); }, cancellationToken ) .ConfigureAwait(false); + } public async ValueTask WriteEntryToAsync( FileInfo destinationFileInfo, @@ -100,7 +104,7 @@ public static class IAsyncReaderExtensions private static async ValueTask CopyEntryToAsync( IAsyncReader reader, Stream writableStream, - int? bufferSize, + ExtractionOptions options, CancellationToken cancellationToken ) { @@ -113,9 +117,14 @@ public static class IAsyncReaderExtensions .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); #endif - var sourceStream = WrapWithProgress(entryStream, reader.Entry); + var checkedStream = IEntryExtensions.WrapWithChecksumValidation( + reader.Entry, + entryStream, + options + ); + var sourceStream = WrapWithProgress(checkedStream, reader.Entry); await sourceStream - .CopyToAsync(writableStream, bufferSize ?? Constants.BufferSize, cancellationToken) + .CopyToAsync(writableStream, options.BufferSize, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Readers/IReaderExtensions.cs b/src/SharpCompress/Readers/IReaderExtensions.cs index 93c391d2..8eb97193 100644 --- a/src/SharpCompress/Readers/IReaderExtensions.cs +++ b/src/SharpCompress/Readers/IReaderExtensions.cs @@ -51,26 +51,35 @@ public static class IReaderExtensions /// /// Extract to specific file /// - public void WriteEntryToFile( - string destinationFileName, - ExtractionOptions? options = null - ) => + public void WriteEntryToFile(string destinationFileName, ExtractionOptions? options = null) + { + options ??= new ExtractionOptions(); reader.Entry.WriteEntryToFile( destinationFileName, options, (x, fm) => { using var fs = File.Open(x, fm); - CopyEntryTo(reader, fs, options?.BufferSize ?? Constants.BufferSize); + CopyEntryTo(reader, fs, options ?? new ExtractionOptions()); } ); + } } - private static void CopyEntryTo(IReader reader, Stream writableStream, int bufferSize) + private static void CopyEntryTo( + IReader reader, + Stream writableStream, + ExtractionOptions options + ) { using var entryStream = reader.OpenEntryStream(); - var sourceStream = WrapWithProgress(entryStream, reader.Entry); - sourceStream.CopyTo(writableStream, bufferSize); + var checkedStream = IEntryExtensions.WrapWithChecksumValidation( + reader.Entry, + entryStream, + options + ); + var sourceStream = WrapWithProgress(checkedStream, reader.Entry); + sourceStream.CopyTo(writableStream, options.BufferSize); } private static Stream WrapWithProgress(Stream source, IEntry entry) diff --git a/tests/SharpCompress.Test/OptionsUsabilityTests.cs b/tests/SharpCompress.Test/OptionsUsabilityTests.cs index 8b08b265..1902e4ab 100644 --- a/tests/SharpCompress.Test/OptionsUsabilityTests.cs +++ b/tests/SharpCompress.Test/OptionsUsabilityTests.cs @@ -306,6 +306,7 @@ public class OptionsUsabilityTests : TestBase Assert.True(preserveMetadata.PreserveAttributes); Assert.Equal(Constants.BufferSize, new ExtractionOptions().BufferSize); + Assert.True(new ExtractionOptions().CheckCrc); } [Fact] diff --git a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs new file mode 100644 index 00000000..14a55fc6 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs @@ -0,0 +1,196 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Writers; +using SharpCompress.Writers.Zip; +using Xunit; + +namespace SharpCompress.Test.Zip; + +public class ZipCrcExtractionTests : ArchiveTests +{ + private const string EntryName = "crc.txt"; + private static readonly byte[] EntryData = Encoding.UTF8.GetBytes("crc validation payload"); + + [Fact] + public void Zip_Archive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-disabled.txt"); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, File.ReadAllBytes(destination)); + } + + [Fact] + public void Zip_Reader_WriteEntryToFile_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var reader = ReaderFactory.OpenReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-mismatch.txt"); + + Assert.True(reader.MoveToNextEntry()); + var exception = Assert.Throws(() => + reader.WriteEntryToFile(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Reader_WriteEntryToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var reader = ReaderFactory.OpenReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-disabled.txt"); + + Assert.True(reader.MoveToNextEntry()); + reader.WriteEntryToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, File.ReadAllBytes(destination)); + } + + [Fact] + public async Task Zip_Archive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-crc-mismatch-async.txt"); + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public async Task Zip_Reader_WriteEntryToFileAsync_Throws_On_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + await using var reader = await ReaderFactory.OpenAsyncReader(zipStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-reader-crc-mismatch-async.txt"); + + Assert.True(await reader.MoveToNextEntryAsync()); + var exception = await Assert.ThrowsAsync(async () => + await reader.WriteEntryToFileAsync(destination) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteToFile_Throws_On_DataDescriptor_Crc_Mismatch() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: true); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + var destination = Path.Combine(SCRATCH_FILES_PATH, "zip-dd-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(EntryName, exception.Message); + } + + private static MemoryStream CreateZipWithInvalidCrc(bool useDataDescriptor) + { + var zipStream = new MemoryStream(); + Stream writerStream = useDataDescriptor ? new NonSeekableWriteStream(zipStream) : zipStream; + using ( + var writer = WriterFactory.OpenWriter( + writerStream, + ArchiveType.Zip, + new ZipWriterOptions(CompressionType.Deflate) { LeaveStreamOpen = true } + ) + ) + { + writer.Write(EntryName, new MemoryStream(EntryData)); + } + + var bytes = zipStream.ToArray(); + CorruptCrc(bytes, ZipHeaderFactoryEntrySignature, 14); + CorruptCrc(bytes, ZipHeaderFactoryDirectorySignature, 16); + return new MemoryStream(bytes); + } + + private const uint ZipHeaderFactoryEntrySignature = 0x04034b50; + private const uint ZipHeaderFactoryDirectorySignature = 0x02014b50; + + private static void CorruptCrc(byte[] bytes, uint signature, int crcOffset) + { + var offset = FindSignature(bytes, signature); + var crcIndex = offset + crcOffset; + bytes[crcIndex] ^= 0xFF; + } + + private static int FindSignature(byte[] bytes, uint signature) + { + var signatureBytes = BitConverter.GetBytes(signature); + for (var i = 0; i <= bytes.Length - signatureBytes.Length; i++) + { + if (bytes.AsSpan(i, signatureBytes.Length).SequenceEqual(signatureBytes)) + { + return i; + } + } + + throw new InvalidOperationException($"ZIP signature 0x{signature:X8} was not found."); + } + + private sealed class NonSeekableWriteStream(Stream stream) : Stream + { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() => stream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + stream.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + stream.Write(buffer, offset, count); + +#if !LEGACY_DOTNET + public override void Write(ReadOnlySpan buffer) => stream.Write(buffer); +#endif + } +} From e9e05520d6b3e18cf81fc7ee33d95da3384d9201 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 15 Jun 2026 09:29:50 +0100 Subject: [PATCH 08/12] 7Zip and Rar --- .../Archives/Rar/RarArchiveEntry.cs | 17 ++-- .../Common/SevenZip/SevenZipEntry.cs | 22 +++++ .../Rar/RarCrcExtractionTests.cs | 36 +++++++++ .../SevenZip/SevenZipCrcExtractionTests.cs | 81 +++++++++++++++++++ 4 files changed, 146 insertions(+), 10 deletions(-) create mode 100644 tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs create mode 100644 tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index d147264e..021381c3 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -71,22 +71,19 @@ public partial class RarArchiveEntry : RarEntry, IArchiveEntry public Stream OpenEntryStream() { + var readStream = new MultiVolumeReadOnlyStream(Parts.Cast()); RarStream stream; if (IsRarV3) { - stream = new RarStream( - archive.UnpackV1.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast()) - ); + stream = RarCrcStream.Create(archive.UnpackV1.Value, FileHeader, readStream); + } + else if (FileHeader.FileCrc?.Length > 5) + { + stream = RarBLAKE2spStream.Create(archive.UnpackV2017.Value, FileHeader, readStream); } else { - stream = new RarStream( - archive.UnpackV2017.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast()) - ); + stream = RarCrcStream.Create(archive.UnpackV2017.Value, FileHeader, readStream); } stream.Initialize(); diff --git a/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs b/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs index fb6d23fa..576c1d16 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipEntry.cs @@ -18,6 +18,28 @@ public class SevenZipEntry : Entry public override long Crc => FilePart.Header.Crc ?? 0; + internal override ChecksumDescriptor Checksum + { + get + { + if ( + IsDirectory + || FilePart.Header.IsAnti + || !FilePart.Header.HasStream + || !FilePart.Header.Crc.HasValue + ) + { + return default; + } + + return new ChecksumDescriptor( + ChecksumKind.Crc32, + FilePart.Header.Crc.Value, + IsAvailable: true + ); + } + } + public override string? Key => FilePart.Header.Name; public override string? LinkTarget => null; diff --git a/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs b/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs new file mode 100644 index 00000000..f2dc8f4f --- /dev/null +++ b/tests/SharpCompress.Test/Rar/RarCrcExtractionTests.cs @@ -0,0 +1,36 @@ +using System.IO; +using System.Linq; +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Common; +using Xunit; + +namespace SharpCompress.Test.Rar; + +public class RarCrcExtractionTests : ArchiveTests +{ + [Theory] + [InlineData("Rar.rar")] + [InlineData("Rar5.rar")] + public void Rar_Archive_WriteToFile_Throws_On_Crc_Mismatch(string archiveName) + { + using var archive = RarArchive.OpenArchive(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, $"{archiveName}-crc-mismatch.txt"); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + private static RarArchiveEntry CorruptFirstFileCrc(IArchive archive) + { + var entry = archive.Entries.OfType().First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static void CorruptCrc(RarArchiveEntry entry) + { + var crc = entry.FileHeader.FileCrc.NotNull(); + crc[0] ^= 0xFF; + } +} diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs new file mode 100644 index 00000000..3261f16a --- /dev/null +++ b/tests/SharpCompress.Test/SevenZip/SevenZipCrcExtractionTests.cs @@ -0,0 +1,81 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Common; +using SharpCompress.Common.SevenZip; +using Xunit; + +namespace SharpCompress.Test.SevenZip; + +public class SevenZipCrcExtractionTests : ArchiveTests +{ + [Fact] + public void SevenZip_Archive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var archive = SevenZipArchive.OpenArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-mismatch.txt"); + + var exception = Assert.Throws(() => entry.WriteToFile(destination)); + + Assert.Contains(entry.Key!, exception.Message); + } + + [Fact] + public void SevenZip_Archive_WriteToFile_Skips_Crc_Mismatch_When_Disabled() + { + using var archive = SevenZipArchive.OpenArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entry = CorruptFirstFileCrc(archive); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-disabled.txt"); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.True(new FileInfo(destination).Length > 0); + } + + [Fact] + public async Task SevenZip_Archive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { + await using var archive = await SevenZipArchive.OpenAsyncArchive( + Path.Combine(TEST_ARCHIVES_PATH, "7Zip.LZMA.7z") + ); + var entries = await archive.EntriesAsync.ToListAsync(); + var entry = CorruptFirstFileCrc(entries); + var destination = Path.Combine(SCRATCH_FILES_PATH, "7zip-crc-mismatch-async.txt"); + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + + Assert.Contains(entry.Key!, exception.Message); + } + + private static IArchiveEntry CorruptFirstFileCrc(IArchive archive) + { + var entry = archive.Entries.First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static IArchiveEntry CorruptFirstFileCrc( + System.Collections.Generic.IEnumerable entries + ) + { + var entry = entries.First(e => !e.IsDirectory); + CorruptCrc(entry); + return entry; + } + + private static void CorruptCrc(IArchiveEntry entry) + { + var sevenZipEntry = Assert.IsAssignableFrom(entry); + var crc = sevenZipEntry.FilePart.Header.Crc.NotNull(); + sevenZipEntry.FilePart.Header.Crc = crc ^ 0xFFFFFFFF; + } +} From 808524d917b3539726e2cbe87dbf1c4f4749d2ce Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 15 Jun 2026 09:59:48 +0100 Subject: [PATCH 09/12] BZip2, GZip and XZ done --- src/SharpCompress/Common/Entry.cs | 12 ++ .../GZip/GZipChecksumValidationStream.cs | 165 ++++++++++++++++++ src/SharpCompress/Common/GZip/GZipEntry.cs | 3 + src/SharpCompress/Common/GZip/GZipFilePart.cs | 6 +- src/SharpCompress/Common/IEntryExtensions.cs | 9 +- .../BZip2/BZip2StreamTests.cs | 45 +++++ .../GZip/GZipCrcExtractionTests.cs | 90 ++++++++++ tests/SharpCompress.Test/Xz/XZStreamTests.cs | 12 ++ 8 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs create mode 100644 tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs create mode 100644 tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs diff --git a/src/SharpCompress/Common/Entry.cs b/src/SharpCompress/Common/Entry.cs index 6c724c32..43095b71 100644 --- a/src/SharpCompress/Common/Entry.cs +++ b/src/SharpCompress/Common/Entry.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using SharpCompress.Common.Options; @@ -86,6 +87,17 @@ public abstract class Entry : IEntry internal virtual ChecksumDescriptor Checksum => default; + internal virtual Stream WrapWithChecksumValidation(Stream source, ExtractionOptions options) + { + var checksum = Checksum; + if (!checksum.IsAvailable) + { + return source; + } + + return new ChecksumValidationStream(source, checksum, Key); + } + /// /// Entry file attribute. /// diff --git a/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs new file mode 100644 index 00000000..512d7cc2 --- /dev/null +++ b/src/SharpCompress/Common/GZip/GZipChecksumValidationStream.cs @@ -0,0 +1,165 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Crypto; + +namespace SharpCompress.Common.GZip; + +internal sealed class GZipChecksumValidationStream : Stream +{ + private readonly Stream _source; + private readonly Stream _rawStream; + private readonly string _entryName; + private readonly uint? _expectedCrc; + private readonly uint? _expectedSize; + private readonly uint[] _crc32Table; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private uint _size; + private bool _validated; + + internal GZipChecksumValidationStream( + Stream source, + Stream rawStream, + string? entryName, + uint? expectedCrc, + uint? expectedSize + ) + { + _source = source; + _rawStream = rawStream; + _entryName = string.IsNullOrEmpty(entryName) ? "Entry" : entryName!; + _expectedCrc = expectedCrc; + _expectedSize = expectedSize; + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + } + + public override bool CanRead => _source.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _source.Length; + + public override long Position + { + get => _source.Position; + set => throw new NotSupportedException(); + } + + public override void Flush() => _source.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _source.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _source.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _source.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } +#endif + + public override int ReadByte() + { + var value = _source.ReadByte(); + if (value == -1) + { + Validate(); + } + else + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, (byte)value); + _size++; + } + + return value; + } + + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _source + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (read > 0) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + _size += unchecked((uint)read); + return; + } + + Validate(); + } + + private void Validate() + { + if (_validated) + { + return; + } + + _validated = true; + + var expectedCrc = _expectedCrc; + var expectedSize = _expectedSize; + if (!expectedCrc.HasValue || !expectedSize.HasValue) + { + Span trailer = stackalloc byte[8]; + _rawStream.ReadFully(trailer); + expectedCrc = BinaryPrimitives.ReadUInt32LittleEndian(trailer); + expectedSize = BinaryPrimitives.ReadUInt32LittleEndian(trailer[4..]); + } + + var actualCrc = ~_seed; + if (actualCrc != expectedCrc.Value) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expectedCrc.Value:X8}, actual 0x{actualCrc:X8}." + ); + } + + if (_size != expectedSize.Value) + { + throw new InvalidFormatException( + $"Size mismatch for entry '{_entryName}'. Expected {expectedSize.Value}, actual {_size}." + ); + } + } +} diff --git a/src/SharpCompress/Common/GZip/GZipEntry.cs b/src/SharpCompress/Common/GZip/GZipEntry.cs index 635fc4cd..48f08d46 100644 --- a/src/SharpCompress/Common/GZip/GZipEntry.cs +++ b/src/SharpCompress/Common/GZip/GZipEntry.cs @@ -20,6 +20,9 @@ public partial class GZipEntry : Entry public override long Crc => _filePart?.Crc ?? 0; + internal override Stream WrapWithChecksumValidation(Stream source, ExtractionOptions options) => + _filePart?.WrapWithChecksumValidation(source, Key) ?? source; + public override string? Key => _filePart?.FilePartName; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/GZip/GZipFilePart.cs b/src/SharpCompress/Common/GZip/GZipFilePart.cs index e8d89a5d..13a1c968 100644 --- a/src/SharpCompress/Common/GZip/GZipFilePart.cs +++ b/src/SharpCompress/Common/GZip/GZipFilePart.cs @@ -5,6 +5,7 @@ using System.IO; using SharpCompress.Common.Tar.Headers; using SharpCompress.Compressors; using SharpCompress.Compressors.Deflate; +using SharpCompress.IO; using SharpCompress.Providers; namespace SharpCompress.Common.GZip; @@ -48,7 +49,7 @@ internal sealed partial class GZipFilePart : FilePart ) : base(archiveEncoding) { - _stream = stream; + _stream = SharpCompressStream.Create(stream); _compressionProviders = compressionProviders; } @@ -66,6 +67,9 @@ internal sealed partial class GZipFilePart : FilePart return _compressionProviders.CreateDecompressStream(CompressionType.Deflate, _stream); } + internal Stream WrapWithChecksumValidation(Stream source, string? entryName) => + new GZipChecksumValidationStream(source, _stream, entryName, Crc, UncompressedSize); + internal override Stream GetRawStream() => _stream; private void ReadTrailer() diff --git a/src/SharpCompress/Common/IEntryExtensions.cs b/src/SharpCompress/Common/IEntryExtensions.cs index 84645910..492ad85d 100644 --- a/src/SharpCompress/Common/IEntryExtensions.cs +++ b/src/SharpCompress/Common/IEntryExtensions.cs @@ -11,13 +11,10 @@ internal static partial class IEntryExtensions ExtractionOptions? options ) { - if (options?.CheckCrc != false && entry is Entry typedEntry) + options ??= new ExtractionOptions(); + if (options.CheckCrc && entry is Entry typedEntry) { - var checksum = typedEntry.Checksum; - if (checksum.IsAvailable) - { - return new ChecksumValidationStream(source, checksum, entry.Key); - } + return typedEntry.WrapWithChecksumValidation(source, options); } return source; diff --git a/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs b/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs new file mode 100644 index 00000000..9bd7f44a --- /dev/null +++ b/tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs @@ -0,0 +1,45 @@ +using System.IO; +using System.Text; +using SharpCompress.Common; +using SharpCompress.Compressors.BZip2; +using Xunit; + +namespace SharpCompress.Test.BZip2; + +public class BZip2StreamTests +{ + [Fact] + public void BZip2Stream_Throws_On_Corrupt_Checksum() + { + var compressed = Compress("BZip2 checksum validation test data."); + compressed[^5] ^= 1; + + using var stream = BZip2Stream.Create( + new MemoryStream(compressed), + SharpCompress.Compressors.CompressionMode.Decompress, + false + ); + using var output = new MemoryStream(); + + Assert.Throws(() => stream.CopyTo(output)); + } + + private static byte[] Compress(string value) + { + using var memoryStream = new MemoryStream(); + using ( + var bzip2Stream = BZip2Stream.Create( + memoryStream, + SharpCompress.Compressors.CompressionMode.Compress, + false, + leaveOpen: true + ) + ) + { + var bytes = Encoding.ASCII.GetBytes(value); + bzip2Stream.Write(bytes, 0, bytes.Length); + } + + return memoryStream.ToArray(); + } +} diff --git a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs new file mode 100644 index 00000000..4f908b3c --- /dev/null +++ b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs @@ -0,0 +1,90 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.GZip; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Readers.GZip; +using SharpCompress.Test.Mocks; +using Xunit; + +namespace SharpCompress.Test.GZip; + +public class GZipCrcExtractionTests : TestBase +{ + [Fact] + public void GZipArchive_WriteToFile_Throws_On_Crc_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + [Fact] + public void GZipArchive_WriteToFile_Throws_On_Size_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: false)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.Throws(() => entry.WriteToFile(destination)); + } + + [Fact] + public void GZipReader_WriteEntryToFile_Throws_On_NonSeekable_Crc_Mismatch() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var nonSeekableStream = new ForwardOnlyStream(stream); + using var reader = GZipReader.OpenReader(nonSeekableStream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + Assert.True(reader.MoveToNextEntry()); + Assert.Throws(() => reader.WriteEntryToFile(destination)); + } + + [Fact] + public void GZipArchive_WriteToFile_Skips_Trailer_Validation_When_CheckCrc_Is_False() + { + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + using var archive = GZipArchive.OpenArchive(stream); + var entry = archive.Entries.Single(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + entry.WriteToFile(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal( + new FileInfo(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar")).Length, + new FileInfo(destination).Length + ); + } + + [Fact] + public async Task GZipArchive_WriteToFileAsync_Throws_On_Crc_Mismatch() + { + await using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); + await using var archive = await GZipArchive.OpenAsyncArchive(stream); + var entry = await archive.EntriesAsync.SingleAsync(); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + + await Assert.ThrowsAsync(async () => + await entry.WriteToFileAsync(destination) + ); + } + + private static byte[] ReadCorruptedGZipTrailer(bool corruptCrc) + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz")); + var trailer = bytes.AsSpan(bytes.Length - 8); + var offset = corruptCrc ? 0 : 4; + var value = BinaryPrimitives.ReadUInt32LittleEndian(trailer[offset..]); + BinaryPrimitives.WriteUInt32LittleEndian(trailer[offset..], value + 1); + return bytes; + } +} diff --git a/tests/SharpCompress.Test/Xz/XZStreamTests.cs b/tests/SharpCompress.Test/Xz/XZStreamTests.cs index 80ad5dc3..00cd2f27 100644 --- a/tests/SharpCompress.Test/Xz/XZStreamTests.cs +++ b/tests/SharpCompress.Test/Xz/XZStreamTests.cs @@ -1,4 +1,5 @@ using System.IO; +using SharpCompress.Common; using SharpCompress.Compressors.Xz; using SharpCompress.IO; using SharpCompress.Test.Mocks; @@ -54,4 +55,15 @@ public class XzStreamTests : XzTestsBase var uncompressed = sr.ReadToEnd(); Assert.Equal(OriginalEmpty, uncompressed); } + + [Fact] + public void Throws_On_Corrupt_Block_Check() + { + var compressed = (byte[])Compressed.Clone(); + compressed[compressed.Length - 29] ^= 1; + using var xz = new XZStream(new MemoryStream(compressed)); + using var output = new MemoryStream(); + + Assert.Throws(() => xz.CopyTo(output)); + } } From e8ee05475795ba9fdc71f2b11c12d22c717fc1ca Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 15 Jun 2026 17:02:31 +0100 Subject: [PATCH 10/12] rest of the formats --- src/SharpCompress/Common/Ace/AceEntry.cs | 8 + src/SharpCompress/Common/Arc/ArcEntry.cs | 5 + src/SharpCompress/Common/Arj/ArjEntry.cs | 7 + .../Common/ChecksumDescriptor.cs | 2 + .../Common/ChecksumValidationStream.cs | 61 +++++- .../Compressors/LZMA/LZipStream.Async.cs | 30 ++- .../Compressors/LZMA/LZipStream.cs | 202 +++++++++++++++++- .../Compressors/LZMA/LzmaStream.cs | 2 + src/SharpCompress/IO/CountingStream.cs | 62 +++++- .../GZip/GZipCrcExtractionTests.cs | 4 + .../RemainingCrcExtractionTests.cs | 75 +++++++ 11 files changed, 442 insertions(+), 16 deletions(-) create mode 100644 tests/SharpCompress.Test/RemainingCrcExtractionTests.cs diff --git a/src/SharpCompress/Common/Ace/AceEntry.cs b/src/SharpCompress/Common/Ace/AceEntry.cs index 419329de..0164d229 100644 --- a/src/SharpCompress/Common/Ace/AceEntry.cs +++ b/src/SharpCompress/Common/Ace/AceEntry.cs @@ -31,6 +31,14 @@ public class AceEntry : Entry } } + internal override ChecksumDescriptor Checksum => + !IsDirectory + && !IsEncrypted + && !_filePart.Header.IsContinuedFromPrev + && !_filePart.Header.IsContinuedToNext + ? new ChecksumDescriptor(ChecksumKind.Crc32NoFinalXor, _filePart.Header.Crc32, true) + : default; + public override string? Key => _filePart?.Header.Filename; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arc/ArcEntry.cs b/src/SharpCompress/Common/Arc/ArcEntry.cs index cb7b262c..9627606d 100644 --- a/src/SharpCompress/Common/Arc/ArcEntry.cs +++ b/src/SharpCompress/Common/Arc/ArcEntry.cs @@ -32,6 +32,11 @@ public class ArcEntry : Entry } } + internal override ChecksumDescriptor Checksum => + _filePart is not null && _filePart.Header.CompressionMethod != CompressionType.Unknown + ? new ChecksumDescriptor(ChecksumKind.Crc16Arc, _filePart.Header.Crc16, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arj/ArjEntry.cs b/src/SharpCompress/Common/Arj/ArjEntry.cs index cf5e1c9f..f0159a66 100644 --- a/src/SharpCompress/Common/Arj/ArjEntry.cs +++ b/src/SharpCompress/Common/Arj/ArjEntry.cs @@ -21,6 +21,13 @@ public class ArjEntry : Entry public override long Crc => _filePart.Header.OriginalCrc32; + internal override ChecksumDescriptor Checksum => + !IsDirectory + && _filePart.Header.CompressionMethod != CompressionMethod.NoDataNoCrc + && _filePart.Header.CompressionMethod != CompressionMethod.NoData + ? new ChecksumDescriptor(ChecksumKind.Crc32, _filePart.Header.OriginalCrc32, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/ChecksumDescriptor.cs b/src/SharpCompress/Common/ChecksumDescriptor.cs index 2dbb6a45..b418df2b 100644 --- a/src/SharpCompress/Common/ChecksumDescriptor.cs +++ b/src/SharpCompress/Common/ChecksumDescriptor.cs @@ -3,6 +3,8 @@ namespace SharpCompress.Common; internal enum ChecksumKind { Crc32, + Crc32NoFinalXor, + Crc16Arc, } internal readonly record struct ChecksumDescriptor( diff --git a/src/SharpCompress/Common/ChecksumValidationStream.cs b/src/SharpCompress/Common/ChecksumValidationStream.cs index fdf4ba42..e3c4e657 100644 --- a/src/SharpCompress/Common/ChecksumValidationStream.cs +++ b/src/SharpCompress/Common/ChecksumValidationStream.cs @@ -13,6 +13,7 @@ internal sealed class ChecksumValidationStream : Stream private readonly string _entryName; private readonly uint[] _crc32Table; private uint _seed = Crc32Stream.DEFAULT_SEED; + private ushort _crc16; private bool _validated; internal ChecksumValidationStream(Stream stream, ChecksumDescriptor checksum, string? entryName) @@ -64,7 +65,7 @@ internal sealed class ChecksumValidationStream : Stream } else { - _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, (byte)value); + UpdateChecksum([(byte)value]); } return value; @@ -107,13 +108,27 @@ internal sealed class ChecksumValidationStream : Stream { if (read > 0) { - _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + UpdateChecksum(buffer); return; } Validate(); } + private void UpdateChecksum(ReadOnlySpan buffer) + { + switch (_checksum.Kind) + { + case ChecksumKind.Crc32: + case ChecksumKind.Crc32NoFinalXor: + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + break; + case ChecksumKind.Crc16Arc: + _crc16 = CalculateCrc16Arc(_crc16, buffer); + break; + } + } + private void Validate() { if (_validated) @@ -123,12 +138,23 @@ internal sealed class ChecksumValidationStream : Stream _validated = true; - if (_checksum.Kind != ChecksumKind.Crc32) + switch (_checksum.Kind) { - return; + case ChecksumKind.Crc32: + ValidateCrc32(finalXor: true); + break; + case ChecksumKind.Crc32NoFinalXor: + ValidateCrc32(finalXor: false); + break; + case ChecksumKind.Crc16Arc: + ValidateCrc16Arc(); + break; } + } - var actual = ~_seed; + private void ValidateCrc32(bool finalXor) + { + var actual = finalXor ? ~_seed : _seed; var expected = unchecked((uint)_checksum.ExpectedValue); if (actual != expected) { @@ -137,4 +163,29 @@ internal sealed class ChecksumValidationStream : Stream ); } } + + private void ValidateCrc16Arc() + { + var expected = unchecked((ushort)_checksum.ExpectedValue); + if (_crc16 != expected) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expected:X4}, actual 0x{_crc16:X4}." + ); + } + } + + private static ushort CalculateCrc16Arc(ushort crc, ReadOnlySpan buffer) + { + foreach (var value in buffer) + { + crc ^= value; + for (var i = 0; i < 8; i++) + { + crc = (crc & 1) != 0 ? (ushort)((crc >> 1) ^ 0xA001) : (ushort)(crc >> 1); + } + } + + return crc; + } } diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs index 2cc7fb8d..949fbc97 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs @@ -169,7 +169,7 @@ public sealed partial class LZipStream public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, cancellationToken); + ) => ReadAndValidateAsync(buffer, cancellationToken); #endif /// @@ -180,7 +180,33 @@ public sealed partial class LZipStream int offset, int count, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, offset, count, cancellationToken); + ) => ReadAndValidateAsync(buffer, offset, count, cancellationToken); + + private async Task ReadAndValidateAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + private async ValueTask ReadAndValidateAsync( + Memory buffer, + CancellationToken cancellationToken + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif /// /// Asynchronously writes bytes from a buffer to the current stream. diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.cs index 618f3b57..f2e1abea 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.cs @@ -22,8 +22,18 @@ public sealed partial class LZipStream : Stream, IFinishable { private readonly Stream _stream; private readonly CountingStream? _countingWritableSubStream; + private readonly CountingStream? _countingReadableSubStream; + private readonly uint[]? _crc32Table; + private readonly ulong? _expectedDataSize; + private readonly ulong? _expectedMemberSize; + private readonly bool _skipTrailerValidation; private bool _disposed; private bool _finished; + private bool _trailerValidated; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private ulong _readCount; + private readonly long _memberStartPosition; + private readonly long _compressedDataStartPosition; private long _writeCount; private readonly Stream? _originalStream; @@ -37,13 +47,43 @@ public sealed partial class LZipStream : Stream, IFinishable if (mode == CompressionMode.Decompress) { + _skipTrailerValidation = stream is SharpCompressStream; + _memberStartPosition = stream.CanSeek ? stream.Position : 0; var dSize = ValidateAndReadSize(stream); if (dSize == 0) { throw new InvalidFormatException("Not an LZip stream"); } var properties = GetProperties(dSize); - _stream = LzmaStream.Create(properties, stream, leaveOpen: leaveOpen); + var trailerStream = GetSeekableTrailerStream(stream); + if (trailerStream is not null) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 16; + Span sizeTrailer = stackalloc byte[16]; + trailerStream.ReadFully(sizeTrailer); + _expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer); + _expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer[8..]); + if (_expectedDataSize > long.MaxValue) + { + throw new InvalidFormatException("LZip data size is too large."); + } + trailerStream.Position = position; + } + _compressedDataStartPosition = stream.CanSeek ? stream.Position : 0; + _countingReadableSubStream = new CountingStream( + SharpCompressStream.CreateNonDisposing(stream) + ); + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + _stream = LzmaStream.Create( + properties, + _countingReadableSubStream, + inputSize: -1, + outputSize: _expectedDataSize.HasValue + ? checked((long)_expectedDataSize.Value) + : -1, + leaveOpen: leaveOpen + ); } else { @@ -106,7 +146,7 @@ public sealed partial class LZipStream : Stream, IFinishable { Finish(); _stream.Dispose(); - if (Mode == CompressionMode.Compress && !_leaveOpen) + if (!_leaveOpen) { _originalStream?.Dispose(); } @@ -134,10 +174,29 @@ public sealed partial class LZipStream : Stream, IFinishable set => throw new NotImplementedException(); } - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } - public override int ReadByte() => _stream.ReadByte(); + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value == -1) + { + ValidateTrailer(); + } + else + { + Span buffer = stackalloc byte[1]; + buffer[0] = (byte)value; + UpdateChecksum(buffer); + } + + return value; + } public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); @@ -145,7 +204,12 @@ public sealed partial class LZipStream : Stream, IFinishable #if !LEGACY_DOTNET - public override int Read(Span buffer) => _stream.Read(buffer); + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } public override void Write(ReadOnlySpan buffer) { @@ -246,4 +310,130 @@ public sealed partial class LZipStream : Stream, IFinishable (byte)((dictionarySize >> 16) & 0xff), (byte)((dictionarySize >> 24) & 0xff), ]; + + private static Stream? GetSeekableTrailerStream(Stream stream) + { + while (stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream) + { + stream = sharpCompressStream.BaseStream(); + } + + if (stream is SeekableSharpCompressStream seekableSharpCompressStream) + { + stream = seekableSharpCompressStream.BaseStream(); + } + + return stream is SharpCompressStream ? null + : stream.CanSeek ? stream + : null; + } + + private static Stream? GetPhysicalSeekableStream(Stream stream) + { + while (stream is SharpCompressStream sharpCompressStream) + { + var baseStream = sharpCompressStream.BaseStream(); + if (ReferenceEquals(baseStream, stream) || !baseStream.CanSeek) + { + break; + } + + stream = baseStream; + } + + return stream.CanSeek ? stream : null; + } + + private static bool IsProbeWrapper(Stream stream) => + stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream + && sharpCompressStream.BaseStream() is SharpCompressStream { IsPassthrough: false }; + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (Mode != CompressionMode.Decompress) + { + return; + } + + if (read > 0) + { + UpdateChecksum(buffer); + return; + } + + ValidateTrailer(); + } + + private void UpdateChecksum(ReadOnlySpan buffer) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table.NotNull(), _seed, buffer); + _readCount += (ulong)buffer.Length; + } + + private void ValidateTrailer() + { + if (_trailerValidated || _skipTrailerValidation || Mode != CompressionMode.Decompress) + { + return; + } + + _trailerValidated = true; + + var countingStream = _countingReadableSubStream.NotNull(); + ulong? compressedDataSize = null; + Span trailer = stackalloc byte[20]; + if (_expectedMemberSize.HasValue && countingStream.CanSeek) + { + compressedDataSize = _expectedMemberSize.Value - 26; + countingStream.Position = _compressedDataStartPosition + (long)compressedDataSize.Value; + countingStream.ReadFully(trailer); + } + else if (GetPhysicalSeekableStream(countingStream.WrappedStream) is { } trailerStream) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 20; + trailerStream.ReadFully(trailer); + trailerStream.Position = position; + } + else + { + compressedDataSize = _stream is LzmaStream lzmaStream + ? (ulong)lzmaStream.CompressedBytesRead + : (ulong)countingStream.BytesRead; + if (countingStream.CanSeek) + { + countingStream.Position = + _compressedDataStartPosition + (long)compressedDataSize.Value; + } + countingStream.ReadFully(trailer); + } + + var expectedCrc = BinaryPrimitives.ReadUInt32LittleEndian(trailer); + var expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[4..]); + var expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[12..]); + + var actualCrc = ~_seed; + if (actualCrc != expectedCrc) + { + throw new InvalidFormatException( + $"LZip CRC mismatch. Expected 0x{expectedCrc:X8}, actual 0x{actualCrc:X8}." + ); + } + + if (_readCount != expectedDataSize) + { + throw new InvalidFormatException( + $"LZip data size mismatch. Expected {expectedDataSize}, actual {_readCount}." + ); + } + + var actualMemberSize = compressedDataSize ?? expectedMemberSize - 26; + actualMemberSize += 26; + if (actualMemberSize != expectedMemberSize) + { + throw new InvalidFormatException( + $"LZip member size mismatch. Expected {expectedMemberSize}, actual {actualMemberSize}." + ); + } + } } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 7e2408ab..f45b33a3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -513,4 +513,6 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } public byte[] Properties { get; } = new byte[5]; + + internal long CompressedBytesRead => _inputPosition; } diff --git a/src/SharpCompress/IO/CountingStream.cs b/src/SharpCompress/IO/CountingStream.cs index 5d1263b3..ef545c9c 100644 --- a/src/SharpCompress/IO/CountingStream.cs +++ b/src/SharpCompress/IO/CountingStream.cs @@ -6,11 +6,12 @@ using System.Threading.Tasks; namespace SharpCompress.IO; /// -/// A simple stream wrapper that counts bytes written without buffering. +/// A simple stream wrapper that counts bytes read and written without buffering. /// internal class CountingStream : Stream { private readonly Stream _stream; + private long _bytesRead; private long _bytesWritten; public CountingStream(Stream stream) @@ -18,6 +19,13 @@ internal class CountingStream : Stream _stream = stream ?? throw new ArgumentNullException(nameof(stream)); } + internal Stream WrappedStream => _stream; + + /// + /// Gets the total number of bytes read from this stream. + /// + public long BytesRead => _bytesRead; + /// /// Gets the total number of bytes written to this stream. /// @@ -42,8 +50,32 @@ internal class CountingStream : Stream public override async Task FlushAsync(CancellationToken cancellationToken) => await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + _bytesRead += read; + return read; + } + + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value != -1) + { + _bytesRead++; + } + + return value; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + _bytesRead += read; + return read; + } +#endif public override long Seek(long offset, SeekOrigin origin) => _stream.Seek(offset, origin); @@ -72,7 +104,31 @@ internal class CountingStream : Stream _bytesWritten += count; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + _bytesRead += read; + return read; + } + #if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _bytesRead += read; + return read; + } + public override async ValueTask WriteAsync( ReadOnlyMemory buffer, CancellationToken cancellationToken = default diff --git a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs index 4f908b3c..2e98c962 100644 --- a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs @@ -68,7 +68,11 @@ public class GZipCrcExtractionTests : TestBase [Fact] public async Task GZipArchive_WriteToFileAsync_Throws_On_Crc_Mismatch() { +#if LEGACY_DOTNET + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#else await using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#endif await using var archive = await GZipArchive.OpenAsyncArchive(stream); var entry = await archive.EntriesAsync.SingleAsync(); var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); diff --git a/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs new file mode 100644 index 00000000..45034123 --- /dev/null +++ b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test; + +public class RemainingCrcExtractionTests : TestBase +{ + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Throws_On_Checksum_Mismatch( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + Assert.Throws(() => reader.WriteAllToDirectory(destination)); + } + + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Skips_Checksum_When_CheckCrc_Is_False( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + reader.WriteAllToDirectory(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.True(Directory.GetFiles(destination, "*", SearchOption.AllDirectories).Length > 0); + } + + [Fact] + public void LZipStream_Throws_On_Trailer_Crc_Mismatch() + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.lz")); + bytes[^20] ^= 1; + using var stream = LZipStream.Create( + new MemoryStream(bytes), + SharpCompress.Compressors.CompressionMode.Decompress + ); + using var output = new MemoryStream(); + + Assert.Throws(() => stream.CopyTo(output)); + } + + private static byte[] ReadCorruptedArchive(string archiveName, string payloadMarker) + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + var marker = System.Text.Encoding.ASCII.GetBytes(payloadMarker); + var offset = bytes.AsSpan().IndexOf(marker); + if (offset < 0) + { + throw new InvalidOperationException($"Payload marker '{payloadMarker}' was not found."); + } + + bytes[offset] ^= 1; + return bytes; + } +} From 727d2a79f953437ebd1b72d5eea2e3eaf93d0ccc Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 18 Jun 2026 17:06:16 +0100 Subject: [PATCH 11/12] updates from review --- docs/API.md | 18 ++++++ .../Archives/IArchiveEntryExtensions.cs | 41 ++++++++++++-- .../Common/ChecksumValidationStream.cs | 15 +---- .../Zip/ZipCrcExtractionTests.cs | 56 +++++++++++++++++++ 4 files changed, 112 insertions(+), 18 deletions(-) diff --git a/docs/API.md b/docs/API.md index 79745918..db807f55 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,6 +132,15 @@ using (var archive = ZipArchive.OpenArchive("file.zip")) var firstEntry = archive.Entries.First(); firstEntry.WriteToFile(@"C:\output\file.txt"); + // Extract single entry to a stream with extraction options + using (var outputStream = File.Create(@"C:\output\file.txt")) + { + firstEntry.WriteTo( + outputStream, + new ExtractionOptions { CheckCrc = false } + ); + } + // Get entry stream using (var stream = entry.OpenEntryStream()) { @@ -154,6 +163,15 @@ await using (var asyncArchive = await ZipArchive.OpenAsyncArchive("file.zip")) { await foreach (var entry in asyncArchive.EntriesAsync) { + await using (var outputStream = File.Create(@"C:\output\" + entry.Key)) + { + await entry.WriteToAsync( + outputStream, + new ExtractionOptions { CheckCrc = false }, + cancellationToken: cancellationToken + ); + } + using (var stream = await entry.OpenEntryStreamAsync(cancellationToken)) { // ... diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index a9c3a054..502a419e 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -18,7 +18,19 @@ public static class IArchiveEntryExtensions /// The stream to write the entry content to. /// Optional progress reporter for tracking extraction progress. public void WriteTo(Stream streamToWriteTo, IProgress? progress = null) => - archiveEntry.WriteTo(streamToWriteTo, null, progress: progress); + archiveEntry.WriteTo(streamToWriteTo, bufferSize: null, progress: progress); + + /// + /// Extract entry to the specified stream. + /// + /// The stream to write the entry content to. + /// Options for configuring extraction behavior. + /// Optional progress reporter for tracking extraction progress. + public void WriteTo( + Stream streamToWriteTo, + ExtractionOptions options, + IProgress? progress = null + ) => archiveEntry.WriteTo(streamToWriteTo, options.BufferSize, options, progress); private void WriteTo( Stream streamToWriteTo, @@ -50,8 +62,7 @@ public static class IArchiveEntryExtensions Stream streamToWriteTo, IProgress? progress = null, CancellationToken cancellationToken = default - ) - { + ) => await archiveEntry .WriteToAsync( streamToWriteTo, @@ -60,7 +71,29 @@ public static class IArchiveEntryExtensions cancellationToken: cancellationToken ) .ConfigureAwait(false); - } + + /// + /// Extract entry to the specified stream asynchronously. + /// + /// The stream to write the entry content to. + /// Options for configuring extraction behavior. + /// Optional progress reporter for tracking extraction progress. + /// Cancellation token. + public async ValueTask WriteToAsync( + Stream streamToWriteTo, + ExtractionOptions options, + IProgress? progress = null, + CancellationToken cancellationToken = default + ) => + await archiveEntry + .WriteToAsync( + streamToWriteTo, + options.BufferSize, + options, + progress, + cancellationToken + ) + .ConfigureAwait(false); private async ValueTask WriteToAsync( Stream streamToWriteTo, diff --git a/src/SharpCompress/Common/ChecksumValidationStream.cs b/src/SharpCompress/Common/ChecksumValidationStream.cs index e3c4e657..1b14cc9d 100644 --- a/src/SharpCompress/Common/ChecksumValidationStream.cs +++ b/src/SharpCompress/Common/ChecksumValidationStream.cs @@ -56,20 +56,7 @@ internal sealed class ChecksumValidationStream : Stream } #endif - public override int ReadByte() - { - var value = _stream.ReadByte(); - if (value == -1) - { - Validate(); - } - else - { - UpdateChecksum([(byte)value]); - } - - return value; - } + public override int ReadByte() => throw new NotSupportedException(); public override async Task ReadAsync( byte[] buffer, diff --git a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs index 14a55fc6..3020e70b 100644 --- a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs @@ -45,6 +45,34 @@ public class ZipCrcExtractionTests : ArchiveTests Assert.Equal(EntryData, File.ReadAllBytes(destination)); } + [Fact] + public void Zip_Archive_WriteTo_Throws_On_Crc_Mismatch_When_Enabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + using var destination = new MemoryStream(); + + var exception = Assert.Throws(() => + entry.WriteTo(destination, new ExtractionOptions { CheckCrc = true }) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public void Zip_Archive_WriteTo_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + using var destination = new MemoryStream(); + + entry.WriteTo(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, destination.ToArray()); + } + [Fact] public void Zip_Reader_WriteEntryToFile_Throws_On_Crc_Mismatch() { @@ -88,6 +116,34 @@ public class ZipCrcExtractionTests : ArchiveTests Assert.Contains(EntryName, exception.Message); } + [Fact] + public async Task Zip_Archive_WriteToAsync_Throws_On_Crc_Mismatch_When_Enabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + await using var destination = new MemoryStream(); + + var exception = await Assert.ThrowsAsync(async () => + await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = true }) + ); + + Assert.Contains(EntryName, exception.Message); + } + + [Fact] + public async Task Zip_Archive_WriteToAsync_Skips_Crc_Mismatch_When_Disabled() + { + using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); + using var archive = ZipArchive.OpenArchive(zipStream); + var entry = archive.Entries.Single(e => !e.IsDirectory); + await using var destination = new MemoryStream(); + + await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.Equal(EntryData, destination.ToArray()); + } + [Fact] public async Task Zip_Reader_WriteEntryToFileAsync_Throws_On_Crc_Mismatch() { From 49d00034e4e79cd9aeed4adb4e342ca0265df6a1 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 19 Jun 2026 11:21:39 +0100 Subject: [PATCH 12/12] update tests --- tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs index 3020e70b..cd0e610c 100644 --- a/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipCrcExtractionTests.cs @@ -122,7 +122,11 @@ public class ZipCrcExtractionTests : ArchiveTests using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); using var archive = ZipArchive.OpenArchive(zipStream); var entry = archive.Entries.Single(e => !e.IsDirectory); +#if LEGACY_DOTNET + using var destination = new MemoryStream(); +#else await using var destination = new MemoryStream(); +#endif var exception = await Assert.ThrowsAsync(async () => await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = true }) @@ -137,7 +141,11 @@ public class ZipCrcExtractionTests : ArchiveTests using var zipStream = CreateZipWithInvalidCrc(useDataDescriptor: false); using var archive = ZipArchive.OpenArchive(zipStream); var entry = archive.Entries.Single(e => !e.IsDirectory); +#if LEGACY_DOTNET + using var destination = new MemoryStream(); +#else await using var destination = new MemoryStream(); +#endif await entry.WriteToAsync(destination, new ExtractionOptions { CheckCrc = false });