From e8ee05475795ba9fdc71f2b11c12d22c717fc1ca Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 15 Jun 2026 17:02:31 +0100 Subject: [PATCH] rest of the formats --- src/SharpCompress/Common/Ace/AceEntry.cs | 8 + src/SharpCompress/Common/Arc/ArcEntry.cs | 5 + src/SharpCompress/Common/Arj/ArjEntry.cs | 7 + .../Common/ChecksumDescriptor.cs | 2 + .../Common/ChecksumValidationStream.cs | 61 +++++- .../Compressors/LZMA/LZipStream.Async.cs | 30 ++- .../Compressors/LZMA/LZipStream.cs | 202 +++++++++++++++++- .../Compressors/LZMA/LzmaStream.cs | 2 + src/SharpCompress/IO/CountingStream.cs | 62 +++++- .../GZip/GZipCrcExtractionTests.cs | 4 + .../RemainingCrcExtractionTests.cs | 75 +++++++ 11 files changed, 442 insertions(+), 16 deletions(-) create mode 100644 tests/SharpCompress.Test/RemainingCrcExtractionTests.cs diff --git a/src/SharpCompress/Common/Ace/AceEntry.cs b/src/SharpCompress/Common/Ace/AceEntry.cs index 419329de..0164d229 100644 --- a/src/SharpCompress/Common/Ace/AceEntry.cs +++ b/src/SharpCompress/Common/Ace/AceEntry.cs @@ -31,6 +31,14 @@ public class AceEntry : Entry } } + internal override ChecksumDescriptor Checksum => + !IsDirectory + && !IsEncrypted + && !_filePart.Header.IsContinuedFromPrev + && !_filePart.Header.IsContinuedToNext + ? new ChecksumDescriptor(ChecksumKind.Crc32NoFinalXor, _filePart.Header.Crc32, true) + : default; + public override string? Key => _filePart?.Header.Filename; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arc/ArcEntry.cs b/src/SharpCompress/Common/Arc/ArcEntry.cs index cb7b262c..9627606d 100644 --- a/src/SharpCompress/Common/Arc/ArcEntry.cs +++ b/src/SharpCompress/Common/Arc/ArcEntry.cs @@ -32,6 +32,11 @@ public class ArcEntry : Entry } } + internal override ChecksumDescriptor Checksum => + _filePart is not null && _filePart.Header.CompressionMethod != CompressionType.Unknown + ? new ChecksumDescriptor(ChecksumKind.Crc16Arc, _filePart.Header.Crc16, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/Arj/ArjEntry.cs b/src/SharpCompress/Common/Arj/ArjEntry.cs index cf5e1c9f..f0159a66 100644 --- a/src/SharpCompress/Common/Arj/ArjEntry.cs +++ b/src/SharpCompress/Common/Arj/ArjEntry.cs @@ -21,6 +21,13 @@ public class ArjEntry : Entry public override long Crc => _filePart.Header.OriginalCrc32; + internal override ChecksumDescriptor Checksum => + !IsDirectory + && _filePart.Header.CompressionMethod != CompressionMethod.NoDataNoCrc + && _filePart.Header.CompressionMethod != CompressionMethod.NoData + ? new ChecksumDescriptor(ChecksumKind.Crc32, _filePart.Header.OriginalCrc32, true) + : default; + public override string? Key => _filePart?.Header.Name; public override string? LinkTarget => null; diff --git a/src/SharpCompress/Common/ChecksumDescriptor.cs b/src/SharpCompress/Common/ChecksumDescriptor.cs index 2dbb6a45..b418df2b 100644 --- a/src/SharpCompress/Common/ChecksumDescriptor.cs +++ b/src/SharpCompress/Common/ChecksumDescriptor.cs @@ -3,6 +3,8 @@ namespace SharpCompress.Common; internal enum ChecksumKind { Crc32, + Crc32NoFinalXor, + Crc16Arc, } internal readonly record struct ChecksumDescriptor( diff --git a/src/SharpCompress/Common/ChecksumValidationStream.cs b/src/SharpCompress/Common/ChecksumValidationStream.cs index fdf4ba42..e3c4e657 100644 --- a/src/SharpCompress/Common/ChecksumValidationStream.cs +++ b/src/SharpCompress/Common/ChecksumValidationStream.cs @@ -13,6 +13,7 @@ internal sealed class ChecksumValidationStream : Stream private readonly string _entryName; private readonly uint[] _crc32Table; private uint _seed = Crc32Stream.DEFAULT_SEED; + private ushort _crc16; private bool _validated; internal ChecksumValidationStream(Stream stream, ChecksumDescriptor checksum, string? entryName) @@ -64,7 +65,7 @@ internal sealed class ChecksumValidationStream : Stream } else { - _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, (byte)value); + UpdateChecksum([(byte)value]); } return value; @@ -107,13 +108,27 @@ internal sealed class ChecksumValidationStream : Stream { if (read > 0) { - _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + UpdateChecksum(buffer); return; } Validate(); } + private void UpdateChecksum(ReadOnlySpan buffer) + { + switch (_checksum.Kind) + { + case ChecksumKind.Crc32: + case ChecksumKind.Crc32NoFinalXor: + _seed = Crc32Stream.CalculateCrc(_crc32Table, _seed, buffer); + break; + case ChecksumKind.Crc16Arc: + _crc16 = CalculateCrc16Arc(_crc16, buffer); + break; + } + } + private void Validate() { if (_validated) @@ -123,12 +138,23 @@ internal sealed class ChecksumValidationStream : Stream _validated = true; - if (_checksum.Kind != ChecksumKind.Crc32) + switch (_checksum.Kind) { - return; + case ChecksumKind.Crc32: + ValidateCrc32(finalXor: true); + break; + case ChecksumKind.Crc32NoFinalXor: + ValidateCrc32(finalXor: false); + break; + case ChecksumKind.Crc16Arc: + ValidateCrc16Arc(); + break; } + } - var actual = ~_seed; + private void ValidateCrc32(bool finalXor) + { + var actual = finalXor ? ~_seed : _seed; var expected = unchecked((uint)_checksum.ExpectedValue); if (actual != expected) { @@ -137,4 +163,29 @@ internal sealed class ChecksumValidationStream : Stream ); } } + + private void ValidateCrc16Arc() + { + var expected = unchecked((ushort)_checksum.ExpectedValue); + if (_crc16 != expected) + { + throw new InvalidFormatException( + $"CRC mismatch for entry '{_entryName}'. Expected 0x{expected:X4}, actual 0x{_crc16:X4}." + ); + } + } + + private static ushort CalculateCrc16Arc(ushort crc, ReadOnlySpan buffer) + { + foreach (var value in buffer) + { + crc ^= value; + for (var i = 0; i < 8; i++) + { + crc = (crc & 1) != 0 ? (ushort)((crc >> 1) ^ 0xA001) : (ushort)(crc >> 1); + } + } + + return crc; + } } diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs index 2cc7fb8d..949fbc97 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs @@ -169,7 +169,7 @@ public sealed partial class LZipStream public override ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, cancellationToken); + ) => ReadAndValidateAsync(buffer, cancellationToken); #endif /// @@ -180,7 +180,33 @@ public sealed partial class LZipStream int offset, int count, CancellationToken cancellationToken = default - ) => _stream.ReadAsync(buffer, offset, count, cancellationToken); + ) => ReadAndValidateAsync(buffer, offset, count, cancellationToken); + + private async Task ReadAndValidateAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } + +#if !LEGACY_DOTNET + private async ValueTask ReadAndValidateAsync( + Memory buffer, + CancellationToken cancellationToken + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + UpdateAndValidateAtEof(buffer.Span[..read], read); + return read; + } +#endif /// /// Asynchronously writes bytes from a buffer to the current stream. diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.cs index 618f3b57..f2e1abea 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.cs @@ -22,8 +22,18 @@ public sealed partial class LZipStream : Stream, IFinishable { private readonly Stream _stream; private readonly CountingStream? _countingWritableSubStream; + private readonly CountingStream? _countingReadableSubStream; + private readonly uint[]? _crc32Table; + private readonly ulong? _expectedDataSize; + private readonly ulong? _expectedMemberSize; + private readonly bool _skipTrailerValidation; private bool _disposed; private bool _finished; + private bool _trailerValidated; + private uint _seed = Crc32Stream.DEFAULT_SEED; + private ulong _readCount; + private readonly long _memberStartPosition; + private readonly long _compressedDataStartPosition; private long _writeCount; private readonly Stream? _originalStream; @@ -37,13 +47,43 @@ public sealed partial class LZipStream : Stream, IFinishable if (mode == CompressionMode.Decompress) { + _skipTrailerValidation = stream is SharpCompressStream; + _memberStartPosition = stream.CanSeek ? stream.Position : 0; var dSize = ValidateAndReadSize(stream); if (dSize == 0) { throw new InvalidFormatException("Not an LZip stream"); } var properties = GetProperties(dSize); - _stream = LzmaStream.Create(properties, stream, leaveOpen: leaveOpen); + var trailerStream = GetSeekableTrailerStream(stream); + if (trailerStream is not null) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 16; + Span sizeTrailer = stackalloc byte[16]; + trailerStream.ReadFully(sizeTrailer); + _expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer); + _expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(sizeTrailer[8..]); + if (_expectedDataSize > long.MaxValue) + { + throw new InvalidFormatException("LZip data size is too large."); + } + trailerStream.Position = position; + } + _compressedDataStartPosition = stream.CanSeek ? stream.Position : 0; + _countingReadableSubStream = new CountingStream( + SharpCompressStream.CreateNonDisposing(stream) + ); + _crc32Table = Crc32Stream.InitializeTable(Crc32Stream.DEFAULT_POLYNOMIAL); + _stream = LzmaStream.Create( + properties, + _countingReadableSubStream, + inputSize: -1, + outputSize: _expectedDataSize.HasValue + ? checked((long)_expectedDataSize.Value) + : -1, + leaveOpen: leaveOpen + ); } else { @@ -106,7 +146,7 @@ public sealed partial class LZipStream : Stream, IFinishable { Finish(); _stream.Dispose(); - if (Mode == CompressionMode.Compress && !_leaveOpen) + if (!_leaveOpen) { _originalStream?.Dispose(); } @@ -134,10 +174,29 @@ public sealed partial class LZipStream : Stream, IFinishable set => throw new NotImplementedException(); } - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + UpdateAndValidateAtEof(buffer.AsSpan(offset, read), read); + return read; + } - public override int ReadByte() => _stream.ReadByte(); + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value == -1) + { + ValidateTrailer(); + } + else + { + Span buffer = stackalloc byte[1]; + buffer[0] = (byte)value; + UpdateChecksum(buffer); + } + + return value; + } public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); @@ -145,7 +204,12 @@ public sealed partial class LZipStream : Stream, IFinishable #if !LEGACY_DOTNET - public override int Read(Span buffer) => _stream.Read(buffer); + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + UpdateAndValidateAtEof(buffer[..read], read); + return read; + } public override void Write(ReadOnlySpan buffer) { @@ -246,4 +310,130 @@ public sealed partial class LZipStream : Stream, IFinishable (byte)((dictionarySize >> 16) & 0xff), (byte)((dictionarySize >> 24) & 0xff), ]; + + private static Stream? GetSeekableTrailerStream(Stream stream) + { + while (stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream) + { + stream = sharpCompressStream.BaseStream(); + } + + if (stream is SeekableSharpCompressStream seekableSharpCompressStream) + { + stream = seekableSharpCompressStream.BaseStream(); + } + + return stream is SharpCompressStream ? null + : stream.CanSeek ? stream + : null; + } + + private static Stream? GetPhysicalSeekableStream(Stream stream) + { + while (stream is SharpCompressStream sharpCompressStream) + { + var baseStream = sharpCompressStream.BaseStream(); + if (ReferenceEquals(baseStream, stream) || !baseStream.CanSeek) + { + break; + } + + stream = baseStream; + } + + return stream.CanSeek ? stream : null; + } + + private static bool IsProbeWrapper(Stream stream) => + stream is SharpCompressStream { IsPassthrough: true } sharpCompressStream + && sharpCompressStream.BaseStream() is SharpCompressStream { IsPassthrough: false }; + + private void UpdateAndValidateAtEof(ReadOnlySpan buffer, int read) + { + if (Mode != CompressionMode.Decompress) + { + return; + } + + if (read > 0) + { + UpdateChecksum(buffer); + return; + } + + ValidateTrailer(); + } + + private void UpdateChecksum(ReadOnlySpan buffer) + { + _seed = Crc32Stream.CalculateCrc(_crc32Table.NotNull(), _seed, buffer); + _readCount += (ulong)buffer.Length; + } + + private void ValidateTrailer() + { + if (_trailerValidated || _skipTrailerValidation || Mode != CompressionMode.Decompress) + { + return; + } + + _trailerValidated = true; + + var countingStream = _countingReadableSubStream.NotNull(); + ulong? compressedDataSize = null; + Span trailer = stackalloc byte[20]; + if (_expectedMemberSize.HasValue && countingStream.CanSeek) + { + compressedDataSize = _expectedMemberSize.Value - 26; + countingStream.Position = _compressedDataStartPosition + (long)compressedDataSize.Value; + countingStream.ReadFully(trailer); + } + else if (GetPhysicalSeekableStream(countingStream.WrappedStream) is { } trailerStream) + { + var position = trailerStream.Position; + trailerStream.Position = trailerStream.Length - 20; + trailerStream.ReadFully(trailer); + trailerStream.Position = position; + } + else + { + compressedDataSize = _stream is LzmaStream lzmaStream + ? (ulong)lzmaStream.CompressedBytesRead + : (ulong)countingStream.BytesRead; + if (countingStream.CanSeek) + { + countingStream.Position = + _compressedDataStartPosition + (long)compressedDataSize.Value; + } + countingStream.ReadFully(trailer); + } + + var expectedCrc = BinaryPrimitives.ReadUInt32LittleEndian(trailer); + var expectedDataSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[4..]); + var expectedMemberSize = BinaryPrimitives.ReadUInt64LittleEndian(trailer[12..]); + + var actualCrc = ~_seed; + if (actualCrc != expectedCrc) + { + throw new InvalidFormatException( + $"LZip CRC mismatch. Expected 0x{expectedCrc:X8}, actual 0x{actualCrc:X8}." + ); + } + + if (_readCount != expectedDataSize) + { + throw new InvalidFormatException( + $"LZip data size mismatch. Expected {expectedDataSize}, actual {_readCount}." + ); + } + + var actualMemberSize = compressedDataSize ?? expectedMemberSize - 26; + actualMemberSize += 26; + if (actualMemberSize != expectedMemberSize) + { + throw new InvalidFormatException( + $"LZip member size mismatch. Expected {expectedMemberSize}, actual {actualMemberSize}." + ); + } + } } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 7e2408ab..f45b33a3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -513,4 +513,6 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable } public byte[] Properties { get; } = new byte[5]; + + internal long CompressedBytesRead => _inputPosition; } diff --git a/src/SharpCompress/IO/CountingStream.cs b/src/SharpCompress/IO/CountingStream.cs index 5d1263b3..ef545c9c 100644 --- a/src/SharpCompress/IO/CountingStream.cs +++ b/src/SharpCompress/IO/CountingStream.cs @@ -6,11 +6,12 @@ using System.Threading.Tasks; namespace SharpCompress.IO; /// -/// A simple stream wrapper that counts bytes written without buffering. +/// A simple stream wrapper that counts bytes read and written without buffering. /// internal class CountingStream : Stream { private readonly Stream _stream; + private long _bytesRead; private long _bytesWritten; public CountingStream(Stream stream) @@ -18,6 +19,13 @@ internal class CountingStream : Stream _stream = stream ?? throw new ArgumentNullException(nameof(stream)); } + internal Stream WrappedStream => _stream; + + /// + /// Gets the total number of bytes read from this stream. + /// + public long BytesRead => _bytesRead; + /// /// Gets the total number of bytes written to this stream. /// @@ -42,8 +50,32 @@ internal class CountingStream : Stream public override async Task FlushAsync(CancellationToken cancellationToken) => await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); - public override int Read(byte[] buffer, int offset, int count) => - _stream.Read(buffer, offset, count); + public override int Read(byte[] buffer, int offset, int count) + { + var read = _stream.Read(buffer, offset, count); + _bytesRead += read; + return read; + } + + public override int ReadByte() + { + var value = _stream.ReadByte(); + if (value != -1) + { + _bytesRead++; + } + + return value; + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) + { + var read = _stream.Read(buffer); + _bytesRead += read; + return read; + } +#endif public override long Seek(long offset, SeekOrigin origin) => _stream.Seek(offset, origin); @@ -72,7 +104,31 @@ internal class CountingStream : Stream _bytesWritten += count; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + _bytesRead += read; + return read; + } + #if !LEGACY_DOTNET + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _bytesRead += read; + return read; + } + public override async ValueTask WriteAsync( ReadOnlyMemory buffer, CancellationToken cancellationToken = default diff --git a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs index 4f908b3c..2e98c962 100644 --- a/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipCrcExtractionTests.cs @@ -68,7 +68,11 @@ public class GZipCrcExtractionTests : TestBase [Fact] public async Task GZipArchive_WriteToFileAsync_Throws_On_Crc_Mismatch() { +#if LEGACY_DOTNET + using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#else await using var stream = new MemoryStream(ReadCorruptedGZipTrailer(corruptCrc: true)); +#endif await using var archive = await GZipArchive.OpenAsyncArchive(stream); var entry = await archive.EntriesAsync.SingleAsync(); var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); diff --git a/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs new file mode 100644 index 00000000..45034123 --- /dev/null +++ b/tests/SharpCompress.Test/RemainingCrcExtractionTests.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test; + +public class RemainingCrcExtractionTests : TestBase +{ + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Throws_On_Checksum_Mismatch( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + Assert.Throws(() => reader.WriteAllToDirectory(destination)); + } + + [Theory] + [InlineData("Arj.store.arj", "This")] + [InlineData("Ace.store.ace", "This")] + [InlineData("Arc.uncompressed.arc", "This")] + public void Reader_WriteEntryToFile_Skips_Checksum_When_CheckCrc_Is_False( + string archiveName, + string payloadMarker + ) + { + using var stream = new MemoryStream(ReadCorruptedArchive(archiveName, payloadMarker)); + using var reader = ReaderFactory.OpenReader(stream); + var destination = Path.Combine(SCRATCH_FILES_PATH, Guid.NewGuid().ToString()); + Directory.CreateDirectory(destination); + + reader.WriteAllToDirectory(destination, new ExtractionOptions { CheckCrc = false }); + + Assert.True(Directory.GetFiles(destination, "*", SearchOption.AllDirectories).Length > 0); + } + + [Fact] + public void LZipStream_Throws_On_Trailer_Crc_Mismatch() + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.lz")); + bytes[^20] ^= 1; + using var stream = LZipStream.Create( + new MemoryStream(bytes), + SharpCompress.Compressors.CompressionMode.Decompress + ); + using var output = new MemoryStream(); + + Assert.Throws(() => stream.CopyTo(output)); + } + + private static byte[] ReadCorruptedArchive(string archiveName, string payloadMarker) + { + var bytes = File.ReadAllBytes(Path.Combine(TEST_ARCHIVES_PATH, archiveName)); + var marker = System.Text.Encoding.ASCII.GetBytes(payloadMarker); + var offset = bytes.AsSpan().IndexOf(marker); + if (offset < 0) + { + throw new InvalidOperationException($"Payload marker '{payloadMarker}' was not found."); + } + + bytes[offset] ^= 1; + return bytes; + } +}