From 208d263a08f086b8cfc2de489c10a9f29041744a Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 3 Aug 2026 15:49:01 +0100 Subject: [PATCH 1/3] Fixes for parallel lzma usage --- .../Common/SevenZip/ArchiveDatabase.cs | 49 +++- .../Common/SevenZip/ArchiveReader.Async.cs | 6 +- .../Compressors/LZMA/Lzma2ParallelDecoder.cs | 220 +++++++++++++++--- .../Compressors/LZMA/LzmaStream.Async.cs | 10 + .../Compressors/LZMA/LzmaStream.cs | 31 +++ .../SevenZip/SevenZipArchiveTests.cs | 100 +++++--- .../Streams/LzmaStreamAsyncTests.cs | 51 ++++ .../Streams/LzmaStreamTests.cs | 169 +++++++++++++- 8 files changed, 556 insertions(+), 80 deletions(-) diff --git a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs index f616a77f..612a0b15 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Linq; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.LZMA.Utilities; using SharpCompress.IO; @@ -229,12 +231,11 @@ internal partial class ArchiveDatabase return false; } - // Parallel decoding needs positional (random-access) reads from the source file and - // positional writes to the temp output file across multiple threads; a real, seekable - // file handle is required for that. Streams the archive was opened from may be wrapped - // (buffering, source-stream indirection, etc.), so unwrap looking for the underlying file. - var inputFile = stream as FileStream ?? (stream as IStreamStack)?.GetStream(); - if (inputFile is null) + // A file handle is not sufficient by itself: packStart is an offset in this stream's + // logical byte sequence. Only use positional I/O when that sequence is exactly one file. + // In particular, SourceStream concatenates multipart archives, where a logical offset + // cannot be passed directly to a handle for just one physical part. + if (!TryGetPositionallyEquivalentInputFile(stream, out var inputFile)) { return false; } @@ -261,6 +262,7 @@ internal partial class ArchiveDatabase try { + tempFile.SetLength(unpackSize); Lzma2ParallelDecoder.DecodeBlocksParallel( inputFile.SafeFileHandle, props, @@ -280,6 +282,41 @@ internal partial class ArchiveDatabase decodedStream = tempFile; return true; } + + private static bool TryGetPositionallyEquivalentInputFile( + Stream stream, + [NotNullWhen(true)] out FileStream? inputFile + ) + { + inputFile = null; + + if (stream is FileStream fileStream) + { + inputFile = fileStream; + return true; + } + + // Volume wraps caller-owned streams in this non-disposing passthrough wrapper. It does + // not change positions, so its single SourceStream input can still be used safely. + if (stream is SharpCompressStream { IsPassthrough: true } passthrough) + { + stream = passthrough.BaseStream(); + } + + if (stream is not SourceStream sourceStream) + { + return false; + } + + var streams = sourceStream.Streams.Take(2).ToArray(); + if (streams.Length != 1 || streams[0] is not FileStream sourceFile) + { + return false; + } + + inputFile = sourceFile; + return true; + } #endif // Cache used to avoid re-decoding a solid folder from scratch for every file it contains. diff --git a/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs b/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs index 52964077..45420797 100644 --- a/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs +++ b/src/SharpCompress/Common/SevenZip/ArchiveReader.Async.cs @@ -211,7 +211,11 @@ internal sealed partial class ArchiveReader await outStream .ReadExactAsync(data, 0, data.Length, cancellationToken) .ConfigureAwait(false); - if (outStream.ReadByte() >= 0) + if ( + await outStream + .ReadAsync(new byte[1], 0, 1, cancellationToken) + .ConfigureAwait(false) > 0 + ) { throw new InvalidFormatException("Decoded stream is longer than expected."); } diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs b/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs index 0539c195..cb63751b 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2ParallelDecoder.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -42,6 +43,10 @@ internal static class Lzma2ParallelDecoder // 7-Zip MtDec.h: #define MTDEC_THREADS_MAX 32 internal const int MaxThreads = 32; + // Bound each worker's transient buffers independently of the compressed block size. + internal const int BlockInputBufferSize = 1 << 16; + internal const int BlockOutputBufferSize = 1 << 20; + /// /// Header-only walk of the LZMA2 chunk stream (no LZMA decoding) that locates every /// independent restart point and merges adjacent segments into blocks using the same size @@ -185,16 +190,6 @@ internal static class Lzma2ParallelDecoder ); } - // Guards against a single unmerged segment large enough to overflow a byte[] buffer - // (only possible with an extreme, effectively-pathological run with zero restarts). - foreach (var block in blocks) - { - if (block.PackLen > int.MaxValue - 16 || block.UnpackLen > int.MaxValue - 16) - { - return null; - } - } - return blocks; } catch (Exception) @@ -232,61 +227,210 @@ internal static class Lzma2ParallelDecoder IReadOnlyList blocks, SafeFileHandle outputHandle, int maxDegreeOfParallelism + ) => + DecodeBlocksParallel( + inputHandle, + lzma2Props, + packStart, + blocks, + outputHandle, + maxDegreeOfParallelism, + ArrayPool.Shared + ); + + internal static void DecodeBlocksParallel( + SafeFileHandle inputHandle, + byte[] lzma2Props, + long packStart, + IReadOnlyList blocks, + SafeFileHandle outputHandle, + int maxDegreeOfParallelism, + ArrayPool bufferPool ) { + ThrowHelper.ThrowIfNull(bufferPool); + Parallel.ForEach( blocks, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, block => { - var packBuffer = new byte[block.PackLen]; - ReadFullyAt(inputHandle, packBuffer, packStart + block.PackOffset); - - using var packStream = new MemoryStream(packBuffer, writable: false); + using var packStream = new RandomAccessBlockStream( + inputHandle, + checked(packStart + block.PackOffset), + block.PackLen, + bufferPool + ); using var lzma = LzmaStream.Create( lzma2Props, packStream, - -1, + block.PackLen, block.UnpackLen, leaveOpen: true ); - var buffer = new byte[Math.Min(block.UnpackLen, 1 << 20)]; - long total = 0; - while (total < block.UnpackLen) + var bufferSize = checked( + (int)Math.Min(Math.Max(block.UnpackLen, 1), BlockOutputBufferSize) + ); + var buffer = bufferPool.Rent(bufferSize); + try { - var toRead = (int)Math.Min(buffer.Length, block.UnpackLen - total); - var read = lzma.Read(buffer, 0, toRead); - if (read <= 0) + long total = 0; + while (total < block.UnpackLen) { - throw new EndOfStreamException( - $"LZMA2 block at unpack offset {block.UnpackOffset:N0} ended early after {total:N0}/{block.UnpackLen:N0} bytes." + var toRead = (int)Math.Min(bufferSize, block.UnpackLen - total); + var read = lzma.Read(buffer, 0, toRead); + if (read <= 0) + { + throw new EndOfStreamException( + $"LZMA2 block at unpack offset {block.UnpackOffset:N0} ended early after {total:N0}/{block.UnpackLen:N0} bytes." + ); + } + RandomAccess.Write( + outputHandle, + buffer.AsSpan(0, read), + checked(block.UnpackOffset + total) ); + total += read; } - RandomAccess.Write( - outputHandle, - buffer.AsSpan(0, read), - block.UnpackOffset + total - ); - total += read; + } + finally + { + bufferPool.Return(buffer, clearArray: true); } } ); } - private static void ReadFullyAt(SafeFileHandle handle, byte[] buffer, long fileOffset) + private sealed class RandomAccessBlockStream : Stream { - var total = 0; - while (total < buffer.Length) + private readonly SafeFileHandle _inputHandle; + private readonly long _start; + private readonly long _length; + private readonly ArrayPool _bufferPool; + private byte[]? _buffer; + private long _bufferStart = -1; + private int _bufferLength; + private long _position; + + internal RandomAccessBlockStream( + SafeFileHandle inputHandle, + long start, + long length, + ArrayPool bufferPool + ) { - var read = RandomAccess.Read(handle, buffer.AsSpan(total), fileOffset + total); - if (read <= 0) + ThrowHelper.ThrowIfNegative(length); + + _inputHandle = inputHandle; + _start = start; + _length = length; + _bufferPool = bufferPool; + _buffer = bufferPool.Rent(BlockInputBufferSize); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => + Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var total = 0; + while (!buffer.IsEmpty && _position < _length) { - throw new EndOfStreamException( - "Unexpected end of pack stream while reading an LZMA2 block." - ); + if (!FillBuffer()) + { + break; + } + var bufferOffset = checked((int)(_position - _bufferStart)); + var count = Math.Min(_bufferLength - bufferOffset, buffer.Length); + _buffer!.AsSpan(bufferOffset, count).CopyTo(buffer); + _position += count; + total += count; + buffer = buffer.Slice(count); } - total += read; + return total; + } + + public override int ReadByte() + { + if (_position >= _length) + { + return -1; + } + + if (!FillBuffer()) + { + return -1; + } + var value = _buffer![checked((int)(_position - _bufferStart))]; + _position++; + return value; + } + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && _buffer is not null) + { + _bufferPool.Return(_buffer, clearArray: true); + _buffer = null; + } + + base.Dispose(disposing); + } + + private bool FillBuffer() + { + if ( + _bufferStart >= 0 + && _position >= _bufferStart + && _position - _bufferStart < _bufferLength + ) + { + return true; + } + + var buffer = + _buffer ?? throw new ObjectDisposedException(nameof(RandomAccessBlockStream)); + var requested = (int)Math.Min(buffer.Length, _length - _position); + var total = 0; + while (total < requested) + { + var read = RandomAccess.Read( + _inputHandle, + buffer.AsSpan(total, requested - total), + checked(_start + _position + total) + ); + if (read <= 0) + { + break; + } + total += read; + } + + _bufferStart = _position; + _bufferLength = total; + return total > 0; } } #endif diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index c89fb923..1c314705 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -175,6 +175,11 @@ public partial class LzmaStream CancellationToken cancellationToken ) { + if (count > 0) + { + EnsureReadMode(ReadMode.Asynchronous); + } + if (_endReached) { return 0; @@ -289,6 +294,11 @@ public partial class LzmaStream CancellationToken cancellationToken = default ) { + if (!buffer.IsEmpty) + { + EnsureReadMode(ReadMode.Asynchronous); + } + if (_endReached) { return 0; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 7f296a15..832c0633 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -36,6 +36,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable private readonly Encoder? _encoder; private byte[]? _asyncHeaderBuffer; private bool _isDisposed; + private ReadMode _readMode; + + private enum ReadMode + { + None, + Synchronous, + Asynchronous, + } private LzmaStream( byte[] properties, @@ -249,6 +257,11 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable public override int Read(byte[] buffer, int offset, int count) { + if (count > 0) + { + EnsureReadMode(ReadMode.Synchronous); + } + if (_endReached) { return 0; @@ -344,6 +357,8 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable public override int ReadByte() { + EnsureReadMode(ReadMode.Synchronous); + if (_endReached) { return -1; @@ -423,6 +438,22 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable return value; } + private void EnsureReadMode(ReadMode readMode) + { + if (_readMode == ReadMode.None) + { + _readMode = readMode; + return; + } + + if (_readMode != readMode) + { + throw new InvalidOperationException( + "A LzmaStream cannot mix synchronous and asynchronous reads." + ); + } + } + private void DecodeChunkHeader() { var control = _inputStream!.ReadByte(); diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs index d1a07532..dc3b1c93 100644 --- a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs +++ b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs @@ -452,39 +452,13 @@ public class SevenZipArchiveTests : ArchiveTests [Fact] public void SevenZipArchive_EnableParallelism_OptInEngagesParallelDecodePath() { - // Highly repetitive so it compresses well, and large enough that -- combined with the - // small dictionary below -- the encoder is forced to periodically reset the dictionary, - // producing multiple independently-decodable LZMA2 restart blocks (the only shape - // Lzma2ParallelDecoder can split across threads). - var payload = new byte[20_000_000]; - for (var i = 0; i < payload.Length; i++) - { - payload[i] = (byte)(i % 7); - } - - // A real (file-backed, seekable) archive is required: the parallel decode path only - // engages for a genuine FileStream, never for archives opened from in-memory streams. var archivePath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-optin.7z"); - var writerOptions = new SevenZipWriterOptions(CompressionType.LZMA2) - { - LzmaProperties = new LzmaEncoderProperties( - eos: false, - dictionary: 1 << 20, - numFastBytes: 32 - ), - }; - using (var archiveStream = File.Create(archivePath)) - using (var writer = new SevenZipWriter(archiveStream, writerOptions)) - using (var source = new MemoryStream(payload)) - { - writer.Write("payload.bin", source, DateTime.UtcNow); - } + var payload = CreateParallelDecodeTestArchive(archivePath); Assert.Equal( payload, ReadSinglePayloadEntry( - archivePath, - ReaderOptions.ForFilePath, + () => SevenZipArchive.OpenArchive(archivePath, ReaderOptions.ForFilePath), out var sequentialFolderStreamType ) ); @@ -493,22 +467,80 @@ public class SevenZipArchiveTests : ArchiveTests Assert.Equal( payload, ReadSinglePayloadEntry( - archivePath, - ReaderOptions.ForFilePath.WithEnableParallelism(true), + () => + SevenZipArchive.OpenArchive( + archivePath, + ReaderOptions.ForFilePath.WithEnableParallelism(true) + ), out var parallelFolderStreamType ) ); Assert.Equal(typeof(FileStream), parallelFolderStreamType); } + [Fact] + public void SevenZipArchive_EnableParallelism_MultipartLzma2FallsBackToSequentialDecode() + { + var archivePath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.7z"); + var payload = CreateParallelDecodeTestArchive(archivePath); + var archiveBytes = File.ReadAllBytes(archivePath); + const int splitOffset = 64; + Assert.True(archiveBytes.Length > splitOffset); + + var firstPartPath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.001"); + var secondPartPath = Path.Combine(SCRATCH_FILES_PATH, "parallel-decode-multipart.002"); + File.WriteAllBytes(firstPartPath, archiveBytes[..splitOffset]); + File.WriteAllBytes(secondPartPath, archiveBytes[splitOffset..]); + var parts = new[] { new FileInfo(firstPartPath), new FileInfo(secondPartPath) }; + + Assert.Equal( + payload, + ReadSinglePayloadEntry( + () => + SevenZipArchive.OpenArchive( + parts, + ReaderOptions.ForFilePath.WithEnableParallelism(true) + ), + out var folderStreamType + ) + ); + Assert.NotEqual(typeof(FileStream), folderStreamType); + } + + private static byte[] CreateParallelDecodeTestArchive(string archivePath) + { + // Highly repetitive so it compresses well, and large enough that -- combined with the + // small dictionary below -- the encoder is forced to periodically reset the dictionary, + // producing multiple independently-decodable LZMA2 restart blocks. + var payload = new byte[20_000_000]; + for (var i = 0; i < payload.Length; i++) + { + payload[i] = (byte)(i % 7); + } + + using var archiveStream = File.Create(archivePath); + using var writer = new SevenZipWriter( + archiveStream, + new SevenZipWriterOptions(CompressionType.LZMA2) + { + LzmaProperties = new LzmaEncoderProperties( + eos: false, + dictionary: 1 << 20, + numFastBytes: 32 + ), + } + ); + using var source = new MemoryStream(payload); + writer.Write("payload.bin", source, DateTime.UtcNow); + return payload; + } + private static byte[] ReadSinglePayloadEntry( - string archivePath, - ReaderOptions readerOptions, + Func openArchive, out Type folderStreamType ) { - using var archive = (SevenZipArchive) - SevenZipArchive.OpenArchive(archivePath, readerOptions); + using var archive = Assert.IsType(openArchive()); using var reader = archive.ExtractAllEntries(); var sevenZipReader = Assert.IsType(reader); sevenZipReader.DiagnosticsEnabled = true; diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs index cc41e831..8a467f73 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamAsyncTests.cs @@ -37,6 +37,47 @@ public class LzmaStreamAsyncTests : TestBase Assert.Equal((byte)'X', buffer[0]); } + [Fact] + public async Task LzmaStream_SynchronousReadThenAsynchronousRead_Throws() + { + using var stream = CreateRawLzmaStream(); + var buffer = new byte[1]; + + Assert.Equal(1, stream.Read(buffer, 0, buffer.Length)); + + await Assert.ThrowsAsync(() => + stream.ReadAsync(buffer, 0, buffer.Length) + ); + } + +#if !LEGACY_DOTNET + [Fact] + public async Task LzmaStream_AsynchronousMemoryReadThenSynchronousRead_Throws() + { + using var stream = CreateRawLzmaStream(); + var buffer = new byte[1]; + + Assert.Equal(1, await stream.ReadAsync(buffer.AsMemory())); + + Assert.Throws(() => stream.Read(buffer, 0, buffer.Length)); + } +#endif + + [Fact] + public async Task LzmaStream_ZeroLengthReadsDoNotSelectReadMode() + { + using var stream = CreateRawLzmaStream(); + var empty = Array.Empty(); + + Assert.Equal(0, stream.Read(empty, 0, 0)); + Assert.Equal(0, await stream.ReadAsync(empty, 0, 0)); + Assert.Equal(1, stream.ReadByte()); + + await Assert.ThrowsAsync(() => + stream.ReadAsync(new byte[1], 0, 1) + ); + } + private static byte[] LzmaData { get; } = [ 0x5D, @@ -625,4 +666,14 @@ public class LzmaStreamAsyncTests : TestBase } } } + + private static LzmaStream CreateRawLzmaStream() + { + var input = new MemoryStream(LzmaData); + var properties = new byte[5]; + Buffer.BlockCopy(LzmaData, 0, properties, 0, properties.Length); + var outputSize = BitConverter.ToInt64(LzmaData, 5); + input.Position = 13; + return LzmaStream.Create(properties, input, input.Length - input.Position, outputSize); + } } diff --git a/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs b/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs index 5a0e77cf..887aaa94 100644 --- a/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs +++ b/tests/SharpCompress.Test/Streams/LzmaStreamTests.cs @@ -866,6 +866,7 @@ public class LzmaStreamTests var inputPath = Path.GetTempFileName(); var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); try { File.WriteAllBytes(inputPath, packBytes); @@ -894,12 +895,20 @@ public class LzmaStreamTests 0, blocks, outputFile.SafeFileHandle, - Environment.ProcessorCount + Environment.ProcessorCount, + bufferPool ); } var actual = File.ReadAllBytes(outputPath); Assert.Equal(expected, actual); + Assert.All( + bufferPool.RentRequests, + request => Assert.InRange(request, 1, Lzma2ParallelDecoder.BlockOutputBufferSize) + ); + Assert.Equal(bufferPool.RentRequests.Count, bufferPool.ReturnedLengths.Count); + Assert.Equal(0, bufferPool.OutstandingRentals); + Assert.All(bufferPool.ClearArrayRequests, Assert.True); } finally { @@ -907,5 +916,163 @@ public class LzmaStreamTests File.Delete(outputPath); } } + + [Fact] + public void Lzma2ParallelDecoder_DecodeBlocksParallel_ReturnsPooledBuffersOnFailure() + { + var packedData = new byte[] { 0x01, 0x00, 0x3F, 0xAA, 0xBB }; + var block = new Lzma2Block(0, packedData.Length, 0, 64); + var inputPath = Path.GetTempFileName(); + var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); + + try + { + File.WriteAllBytes(inputPath, packedData); + + using ( + var inputFile = new FileStream( + inputPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ) + ) + using ( + var outputFile = new FileStream( + outputPath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None + ) + ) + { + outputFile.SetLength(block.UnpackLen); + Assert.ThrowsAny(() => + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + Lzma2Props, + 0, + [block], + outputFile.SafeFileHandle, + 1, + bufferPool + ) + ); + } + + Assert.Equal(bufferPool.RentRequests.Count, bufferPool.ReturnedLengths.Count); + Assert.Equal(0, bufferPool.OutstandingRentals); + Assert.All(bufferPool.ClearArrayRequests, Assert.True); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + [Fact] + public void Lzma2ParallelDecoder_DecodeBlocksParallel_SupportsLongPackLengths() + { + var expected = RepeatingPayload(64, 7); + var packedData = UncompressedChunk(dictReset: true, expected); + var block = new Lzma2Block(0, (long)int.MaxValue + 1, 0, expected.Length); + var inputPath = Path.GetTempFileName(); + var outputPath = Path.GetTempFileName(); + var bufferPool = new TrackingArrayPool(); + + try + { + File.WriteAllBytes(inputPath, packedData); + + using ( + var inputFile = new FileStream( + inputPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ) + ) + using ( + var outputFile = new FileStream( + outputPath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None + ) + ) + { + outputFile.SetLength(block.UnpackLen); + Lzma2ParallelDecoder.DecodeBlocksParallel( + inputFile.SafeFileHandle, + Lzma2Props, + 0, + [block], + outputFile.SafeFileHandle, + 1, + bufferPool + ); + } + + Assert.Equal(expected, File.ReadAllBytes(outputPath)); + Assert.All( + bufferPool.RentRequests, + request => Assert.InRange(request, 1, Lzma2ParallelDecoder.BlockOutputBufferSize) + ); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + private sealed class TrackingArrayPool : ArrayPool + { + private readonly object _lock = new(); + private readonly HashSet _rented = new(); + + public List RentRequests { get; } = new(); + public List ReturnedLengths { get; } = new(); + public List ClearArrayRequests { get; } = new(); + + public int OutstandingRentals + { + get + { + lock (_lock) + { + return _rented.Count; + } + } + } + + public override byte[] Rent(int minimumLength) + { + var buffer = new byte[minimumLength]; + lock (_lock) + { + RentRequests.Add(minimumLength); + _rented.Add(buffer); + } + return buffer; + } + + public override void Return(byte[] array, bool clearArray = false) + { + lock (_lock) + { + Assert.True(_rented.Remove(array), "A buffer must be returned only once."); + ReturnedLengths.Add(array.Length); + ClearArrayRequests.Add(clearArray); + } + + if (clearArray) + { + Array.Clear(array, 0, array.Length); + } + } + } #endif } From c619ec068740083ef59742780c36333e51593e0c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 4 Aug 2026 09:49:11 +0100 Subject: [PATCH 2/3] change generated dispose to have a sync only option --- src/SharpCompress/Archives/IArchiveEntryExtensions.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 36912f9e..31b4596a 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -71,12 +71,16 @@ public static partial class IArchiveEntryExtensions throw new ExtractionException("Entry is a file directory and cannot be extracted."); } +#if SYNC_ONLY + using var entryStream = archiveEntry.OpenEntryStream(); +#else var entryStream = await archiveEntry .OpenEntryStreamAsync(cancellationToken) .ConfigureAwait(false); await using var entryStreamScope = entryStream .DisposeAsyncScope() .ConfigureAwait(false); +#endif var checkedStream = options is null ? entryStream : IEntryExtensions.WrapWithChecksumValidation(archiveEntry, entryStream, options); From 46e740b61385e2665fbdd402e7ee101a42310881 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 4 Aug 2026 10:16:09 +0100 Subject: [PATCH 3/3] Remove UseSyncOverAsyncDispose --- src/SharpCompress/Common/EntryStream.cs | 13 +-------- .../Common/Tar/TarReadOnlySubStream.cs | 13 +-------- .../Common/Zip/WinzipAesCryptoStream.cs | 27 +++++++++---------- src/SharpCompress/Utility.cs | 9 ------- 4 files changed, 15 insertions(+), 47 deletions(-) diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index d2e3f51c..bf835b93 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -30,18 +30,7 @@ public partial class EntryStream : AsyncDisposableStream _isDisposed = true; if (!(_completed || _reader.Cancelled)) { - if (Utility.UseSyncOverAsyncDispose()) - { -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits -#pragma warning disable CA2012 - SkipEntryAsync().GetAwaiter().GetResult(); -#pragma warning restore CA2012 -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - else - { - SkipEntry(); - } + SkipEntry(); } //Need a safe standard approach to this - it's okay for compression to overreads. Handling needs to be standardised diff --git a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs index 86594f63..c4dc0b72 100644 --- a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs +++ b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs @@ -29,18 +29,7 @@ internal class TarReadOnlySubStream : Stream _isDisposed = true; if (disposing) { - if (Utility.UseSyncOverAsyncDispose()) - { -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits -#pragma warning disable CA2012 - AdvanceToNextHeaderAsync().GetAwaiter().GetResult(); -#pragma warning restore CA2012 -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - else - { - AdvanceToNextHeader(); - } + AdvanceToNextHeader(); } base.Dispose(disposing); } diff --git a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs index f6343c02..8a4eda18 100644 --- a/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs +++ b/src/SharpCompress/Common/Zip/WinzipAesCryptoStream.cs @@ -69,28 +69,27 @@ internal partial class WinzipAesCryptoStream : Stream _isDisposed = true; if (disposing) { - // Read out last 10 auth bytes - catch exceptions for async-only streams - if (Utility.UseSyncOverAsyncDispose()) + // Read out last 10 auth bytes +#if LEGACY_DOTNET + // Stream has no DisposeAsync on legacy targets, so async flows fall back to this + // sync Dispose while the underlying stream may be async-only. + var ten = ArrayPool.Shared.Rent(10); + try { - var ten = ArrayPool.Shared.Rent(10); - try - { #pragma warning disable VSTHRD002 // Avoid problematic synchronous waits #pragma warning disable CA2012 - _stream.ReadFullyAsync(ten, 0, 10).GetAwaiter().GetResult(); + _stream.ReadFullyAsync(ten, 0, 10).GetAwaiter().GetResult(); #pragma warning restore CA2012 #pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - } - finally - { - ArrayPool.Shared.Return(ten); - } } - else + finally { - Span ten = stackalloc byte[10]; - _stream.ReadFully(ten); + ArrayPool.Shared.Return(ten); } +#else + Span ten = stackalloc byte[10]; + _stream.ReadFully(ten); +#endif _stream.Dispose(); } base.Dispose(disposing); diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index c03c3643..c5824d51 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -23,15 +23,6 @@ internal static partial class Utility ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - public static bool UseSyncOverAsyncDispose() - { - var useSyncOverAsync = false; -#if LEGACY_DOTNET - useSyncOverAsync = true; -#endif - return useSyncOverAsync; - } - private static readonly HashSet invalidChars = new(Path.GetInvalidFileNameChars()); public static ReadOnlyCollection ToReadOnly(this IList items) => new(items);