diff --git a/docs/SEVENZIP_PROFILING.md b/docs/SEVENZIP_PROFILING.md index f4be5aa6..80c07838 100644 --- a/docs/SEVENZIP_PROFILING.md +++ b/docs/SEVENZIP_PROFILING.md @@ -314,3 +314,140 @@ Validation for this follow-up: **466 targeted tests** and **2,997 complete .NET succeed; net48 tests were not run on macOS. Formatting and whitespace checks pass. No public signatures or CRC-validation semantics changed. This phase did not introduce a global pool or increase the production dictionary-growth factor. + +## Follow-up: encoder workspaces and metadata scratch + +This round starts at `03056961`, after the preceding changes. Saved libraries, +benchmark sources, JSON logs and EventPipe traces live in the external artifact +directory's `round3/` subdirectory. Comparisons use Release net10.0, two warmups +and five measured microbenchmark iterations. Large extraction uses the existing +200 MiB solid LZMA2 archive, two warmups and three measured iterations. Inputs, +metadata objects and reflection delegates are prepared outside measurement. +Builds and tests do not run concurrently with measured workloads. + +### Implementation + +- The encoder rents its 4,096-element optimization workspace on first use, + clears its logical range on initialization and returns it on disposal. This + removes a recurring large-object allocation without pooling encoder objects. +- LZMA2 block compression keeps its pooled output stream alive through trial + decoding and chunk writing instead of materializing it with `ToArray()`. + The output is truncated to the trial decoder's consumed byte count, preserving + range-coder flush-byte handling. Uncompressed fallback still works; exceptions + return the temporary output and invalidate failed trial-decoder state. +- A trial decoder is reused within each LZMA2 encoder stream. Fixed scalar + decoder tables initialize only when scalar decoding is actually needed. + Scalar literal probabilities are pooled. Fast probabilities share one pooled + bank containing fixed tables followed by literals, keeping the default bank + within a 16 KiB pool bucket. Initialization uses logical lengths, not capacity. + Both stream-disposal paths now dispose their decoder to return model rentals. +- Direct `LzmaStream.WriteAsync(ReadOnlyMemory)` borrows memory through an + internal read-only adapter until encoding completes, rather than copying it. + This includes memory not backed by an array. The current 7z copy loop calls the + array overload, so this benefit is measured separately from archive writing. +- Synchronous integer, digest, coder-ID, timestamp, attribute and signature-header + scratch uses bounded stack buffers. Folder validation's 32-word stack mask is + explicitly cleared. Async signature headers retain owned storage across awaits. + Legacy span-write compatibility can still rent/copy; it is not a zero-copy claim. +- FilesInfo computes property sizes and writes directly, eliminating temporary + property streams, closures and bool arrays. Names use 512-byte stack scratch + for short names on modern targets and a reusable rented buffer for long names. + Legacy encoding uses a rented buffer. CRC-defined vectors also write directly. + +No public API signatures, dictionary-growth policy, stream ownership defaults +or checksum-validation semantics change. + +### Match-finder bucket experiment rejected + +The binary-tree match finder needs `2 * (dictionarySize + 1)` uint entries. +For a 1 MiB dictionary, its logical payload is 8 MiB + 8 bytes, crossing into a +16 MiB shared-pool bucket on modern .NET. An experimental build replaced this +default rental with an exact-size owned array, retaining custom-pool behavior. +It reduced capacity per active encoder but lost warm buffer reuse: 16 small +files allocated **145,934,592 B for LZMA** and **143,684,800 B for LZMA2**, with +median elapsed times of 20.7 ms and 26.7 ms respectively. This is substantially +worse allocation behavior than the retained shared-pool implementation. + +The exact-array experiment was removed. A bounded exact-size reusable cache +could address both concerns, but was not introduced: it requires a separate +retention/concurrency policy rather than silently replacing shared pooling with +an unbounded second cache or reducing the effective dictionary size. Likewise, +the 17 KiB length-price tables are not independently pooled into 32 KiB buckets; +this round prioritizes the existing large optimization workspace and model bank. + +### Results + +Exact managed allocation deltas are medians after warmup. These results describe +pool reuse, not guaranteed allocation-free cold starts or lower process RSS. +Pool capacity can exceed logical data sizes; oversized literal banks are released +when switching to substantially smaller context properties rather than retained +for the decoder's entire remaining lifetime. + +| Workload | Baseline allocated | Updated allocated | +| --- | ---: | ---: | +| 16 independent 4 KiB files, LZMA writer sync | 6,389,192 B | 3,319,696 B | +| Same files, LZMA writer async | 4,338,864 B | 1,269,480 B | +| Same files, LZMA2 writer sync | 4,582,864 B | 1,071,896 B | +| Same files, LZMA2 writer async | 4,658,968 B | 1,148,112 B | +| Three compressible 2 MiB LZMA2 blocks, sync/async | 307,360 B | 55,144 B | +| Three random 2 MiB LZMA2 blocks, sync/async | 6,684,832 B | 56,704 B | +| Serialize 10,000 mixed metadata entries | 2,868,992 B | 40 B | +| 64 direct 4 KiB LZMA memory writes | 501,360 B | 56,408 B | +| Decode 64 independent reset chunks, sync | 24,072 B | 712 B | +| Decode 64 independent reset chunks, async | 33,472 B | 8,784 B | + +The metadata microbenchmark includes long/short Unicode names, partial times, +directories and empty files. Metadata objects and delegates already exist; it +does not include archive creation, header compression, or public entry objects. +The remaining 40 bytes are the harness stopwatch. The block-compression benchmark +discards output to `Stream.Null`; round-trip tests independently validate emitted +data. The writer benchmark includes finalization and seekable MemoryStream output. + +| 200 MiB extraction | Baseline allocated | Updated allocated | Baseline elapsed | Updated elapsed | +| --- | ---: | ---: | ---: | ---: | +| Archive entries sync | 57,640 B | 9,992 B | 2.612 s | 2.554 s | +| Archive entries async | 76,024 B | 26,344 B | 14.420 s | 14.130 s | + +Allocation savings are approximately **83% sync / 65% async** relative to this +round's already-optimized baseline. These raw entry-stream extractions retain +the earlier report's checksum-validation caveat. + +Writer elapsed times were noisy in the main run. A separate three-process-pair +repeat alternated baseline/final execution order; each process warmed twice and +measured five iterations. Medians across the 15 measured iterations per version: + +| Small-file writer | Baseline elapsed | Updated elapsed | +| --- | ---: | ---: | +| LZMA sync | 18.57 ms | 18.66 ms | +| LZMA async | 34.49 ms | 34.87 ms | +| LZMA2 sync | 22.51 ms | 23.52 ms | +| LZMA2 async | 22.36 ms | 23.15 ms | + +LZMA is essentially unchanged; **small-file LZMA2 costs about 3–5% elapsed time** +in this repeat while allocating 75–77% less. The allocation-focused changes are +retained with that tradeoff explicit, not described as a universal throughput +improvement. Large/block timing variation likewise does not justify a general +speedup claim. + +Separate baseline/final EventPipe sampled-thread-time and allocation traces of +random-block writing report **zero lost events**. Baseline measured-phase samples +include about 31.9 MB of byte arrays and 0.90 MB of `Optimal[]`; the final trace +has no corresponding recurring samples. Remaining samples are sparse. CPU leaves +remain dominated by encoder block processing, binary-tree matching and trial +decoding. Exact allocation deltas are stronger evidence than sampled percentages. + +`run.py` reproduces microbenchmarks, extraction comparisons and traces; +`repeat.py` reproduces the interleaved writer timing check; `exact.py` measures +the rejected exact-array match-finder snapshot. Baseline, intermediate, +consolidated-final and exact-array libraries are saved separately. + +### Validation + +- **485 targeted tests** and **3,016 complete .NET 10 Release tests** pass. +- All six library targets, net48 tests, and the performance project compile + without warnings/errors. net48 tests are not executed on macOS. +- New coverage exercises multiple compressed/uncompressed LZMA2 blocks, partial + decoder disposal, failed raw decoding, pooled workspace disposal, shrinking + context banks, non-array memory writes, cancellation and caller-memory reuse, + long/invalid-surrogate names, partial metadata and digest-vector byte boundaries. +- CSharpier formatting/check and whitespace validation pass. diff --git a/src/SharpCompress/Common/SevenZip/CFolder.cs b/src/SharpCompress/Common/SevenZip/CFolder.cs index 683c27ca..eb95da39 100644 --- a/src/SharpCompress/Common/SevenZip/CFolder.cs +++ b/src/SharpCompress/Common/SevenZip/CFolder.cs @@ -138,7 +138,8 @@ internal class CFolder } } - var mask = new uint[kMaskSize]; + Span mask = stackalloc uint[kMaskSize]; + mask.Clear(); { var inStreamToCoder = new List(); diff --git a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs index b06b13ea..10a832e4 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs @@ -1,8 +1,8 @@ using System; +using System.Buffers; using System.IO; using System.Text; using SharpCompress.Compressors.LZMA.Utilities; -using SharpCompress.IO; namespace SharpCompress.Common.SevenZip; @@ -63,18 +63,13 @@ internal sealed class SevenZipFilesInfoWriter private void WriteEmptyStreamProperty(Stream stream, int emptyStreamCount) { - var emptyStreams = new bool[Entries.Length]; - var emptyFiles = new bool[emptyStreamCount]; var hasEmptyFile = false; - var emptyIndex = 0; for (var i = 0; i < Entries.Length; i++) { if (Entries[i].IsEmpty || Entries[i].IsDirectory) { - emptyStreams[i] = true; var isEmptyFile = !Entries[i].IsDirectory; - emptyFiles[emptyIndex++] = isEmptyFile; if (isEmptyFile) { hasEmptyFile = true; @@ -83,147 +78,194 @@ internal sealed class SevenZipFilesInfoWriter } // kEmptyStream - WriteFileProperty(stream, BlockType.EmptyStream, s => s.WriteBoolVector(emptyStreams)); + WritePropertyHeader(stream, BlockType.EmptyStream, (Entries.LongLength + 7) / 8); + WriteFlags(stream, BlockType.EmptyStream); // kEmptyFile (only if there are actual empty files, not just directories) if (hasEmptyFile) { - WriteFileProperty(stream, BlockType.EmptyFile, s => s.WriteBoolVector(emptyFiles)); + WritePropertyHeader(stream, BlockType.EmptyFile, ((long)emptyStreamCount + 7) / 8); + WriteFlags(stream, BlockType.EmptyFile); } } private void WriteNameProperty(Stream stream) { - WriteFileProperty( - stream, - BlockType.Name, - s => + long size = 1; + var maxBytes = 0; + foreach (var entry in Entries) + { + var count = Encoding.Unicode.GetByteCount(entry.Name); + size = checked(size + count + 2); + maxBytes = Math.Max(maxBytes, count); + } + WritePropertyHeader(stream, BlockType.Name, size); + stream.WriteByte(0); + byte[]? rented = null; +#if !LEGACY_DOTNET + Span small = stackalloc byte[512]; +#endif + try + { + foreach (var entry in Entries) { - // External = 0 (inline) - s.WriteByte(0); - - for (var i = 0; i < Entries.Length; i++) +#if !LEGACY_DOTNET + var count = Encoding.Unicode.GetByteCount(entry.Name); + if (count <= small.Length) { - var nameBytes = Encoding.Unicode.GetBytes(Entries[i].Name); - s.Write(nameBytes); - // null terminator (2 bytes for UTF-16) - s.WriteByte(0); - s.WriteByte(0); + Encoding.Unicode.GetBytes(entry.Name.AsSpan(), small); + stream.Write(small.Slice(0, count)); } + else +#endif + { + rented ??= ArrayPool.Shared.Rent(maxBytes); + var written = Encoding.Unicode.GetBytes( + entry.Name, + 0, + entry.Name.Length, + rented, + 0 + ); + stream.Write(rented, 0, written); + } + stream.WriteByte(0); + stream.WriteByte(0); } - ); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } } private void WriteMTimeProperty(Stream stream) { - var hasTimes = false; + var definedCount = 0; for (var i = 0; i < Entries.Length; i++) { - if (Entries[i].ModificationTime != null) + if (Entries[i].ModificationTime is not null) { - hasTimes = true; - break; + definedCount++; } } - if (!hasTimes) + if (definedCount == 0) { return; } - WriteFileProperty( + WritePropertyHeader( stream, BlockType.MTime, - s => - { - var defined = new bool[Entries.Length]; - for (var i = 0; i < Entries.Length; i++) - { - defined[i] = Entries[i].ModificationTime != null; - } - s.WriteOptionalBoolVector(defined); - - // External = 0 (inline) - s.WriteByte(0); - - var buf = new byte[8]; - for (var i = 0; i < Entries.Length; i++) - { - if (Entries[i].ModificationTime is { } mtime) - { - var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc(); - System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian( - buf, - fileTime - ); - s.Write(buf, 0, 8); - } - } - } + 2L + + (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8) + + 8L * definedCount ); + stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0)); + if (definedCount != Entries.Length) + { + WriteFlags(stream, BlockType.MTime); + } + stream.WriteByte(0); + Span buf = stackalloc byte[8]; + for (var i = 0; i < Entries.Length; i++) + { + if (Entries[i].ModificationTime is { } mtime) + { + var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc(); + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(buf, fileTime); + stream.Write(buf); + } + } } private void WriteAttributesProperty(Stream stream) { - var hasAttrs = false; + var definedCount = 0; for (var i = 0; i < Entries.Length; i++) { - if (Entries[i].Attributes != null) + if (Entries[i].Attributes is not null) { - hasAttrs = true; - break; + definedCount++; } } - if (!hasAttrs) + if (definedCount == 0) { return; } - WriteFileProperty( + WritePropertyHeader( stream, BlockType.WinAttributes, - s => - { - var defined = new bool[Entries.Length]; - for (var i = 0; i < Entries.Length; i++) - { - defined[i] = Entries[i].Attributes != null; - } - s.WriteOptionalBoolVector(defined); - - // External = 0 (inline) - s.WriteByte(0); - - var buf = new byte[4]; - for (var i = 0; i < Entries.Length; i++) - { - if (Entries[i].Attributes is { } attrs) - { - System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs); - s.Write(buf, 0, 4); - } - } - } + 2L + + (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8) + + 4L * definedCount ); + stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0)); + if (definedCount != Entries.Length) + { + WriteFlags(stream, BlockType.WinAttributes); + } + stream.WriteByte(0); + Span buf = stackalloc byte[4]; + for (var i = 0; i < Entries.Length; i++) + { + if (Entries[i].Attributes is { } attrs) + { + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs); + stream.Write(buf); + } + } } /// - /// Writes a file property block: PropertyID + size + data. - /// Size is computed by writing to a temporary buffer first. + /// Writes a property prefix without staging its computable payload in another stream. /// - private static void WriteFileProperty( - Stream stream, - BlockType propertyId, - Action writeData - ) + private static void WritePropertyHeader(Stream stream, BlockType propertyId, long size) { - using var dataStream = new PooledMemoryStream(); - writeData(dataStream); - stream.WriteByte((byte)propertyId); - stream.WriteEncodedUInt64((ulong)dataStream.Length); - dataStream.Position = 0; - dataStream.CopyTo(stream); + stream.WriteEncodedUInt64((ulong)size); + } + + private void WriteFlags(Stream stream, BlockType property) + { + byte mask = 0x80; + byte bits = 0; + foreach (var entry in Entries) + { + var empty = entry.IsEmpty || entry.IsDirectory; + if (property == BlockType.EmptyFile && !empty) + { + continue; + } + var defined = property switch + { + BlockType.EmptyStream => empty, + BlockType.EmptyFile => !entry.IsDirectory, + BlockType.MTime => entry.ModificationTime is not null, + BlockType.WinAttributes => entry.Attributes is not null, + _ => throw new InvalidOperationException(), + }; + if (defined) + { + bits |= mask; + } + mask >>= 1; + if (mask == 0) + { + stream.WriteByte(bits); + mask = 0x80; + bits = 0; + } + } + if (mask != 0x80) + { + stream.WriteByte(bits); + } } } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs b/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs index 24e86bfd..bfbc4718 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs @@ -13,21 +13,43 @@ internal sealed class SevenZipDigestsWriter(uint?[] crcs) public void Write(Stream stream) { - var defined = new bool[CRCs.Length]; + var allDefined = true; for (var i = 0; i < CRCs.Length; i++) { - defined[i] = CRCs[i] != null; + allDefined &= CRCs[i] is not null; + } + stream.WriteByte((byte)(allDefined ? 1 : 0)); + if (!allDefined) + { + byte mask = 0x80; + byte bits = 0; + foreach (var crc in CRCs) + { + if (crc is not null) + { + bits |= mask; + } + mask >>= 1; + if (mask == 0) + { + stream.WriteByte(bits); + mask = 0x80; + bits = 0; + } + } + if (mask != 0x80) + { + stream.WriteByte(bits); + } } - stream.WriteOptionalBoolVector(defined); - - var buf = new byte[4]; + Span buf = stackalloc byte[4]; for (var i = 0; i < CRCs.Length; i++) { if (CRCs[i] is { } crcValue) { System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, crcValue); - stream.Write(buf, 0, 4); + stream.Write(buf); } } } @@ -184,14 +206,14 @@ internal sealed class SevenZipUnPackInfoWriter stream.WriteByte(attributes); // Codec ID bytes (big-endian, most significant byte first) - var codecId = new byte[codecIdLength]; + Span codecId = stackalloc byte[8]; var id = coder._methodId._id; for (var i = codecIdLength - 1; i >= 0; i--) { codecId[i] = (byte)(id & 0xFF); id >>= 8; } - stream.Write(codecId, 0, codecIdLength); + stream.Write(codecId.Slice(0, codecIdLength)); if (isComplex) { diff --git a/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs b/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs index 060bc654..3733cb15 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs @@ -29,17 +29,18 @@ internal static class SevenZipSignatureHeaderWriter /// public static void WritePlaceholder(Stream stream) { - var header = new byte[HeaderSize]; + Span header = stackalloc byte[HeaderSize]; + header.Clear(); // magic signature - Array.Copy(Signature, 0, header, 0, Signature.Length); + Signature.AsSpan().CopyTo(header); // version: major=0, minor=2 (standard 7z format) header[6] = 0; header[7] = 2; // remaining 24 bytes are zero (placeholder for CRC and StartHeader) - stream.Write(header, 0, header.Length); + stream.Write(header); } /// @@ -79,11 +80,12 @@ internal static class SevenZipSignatureHeaderWriter uint nextHeaderCrc ) { - var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc); + Span header = stackalloc byte[HeaderSize]; + BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc); // Write at position 0 stream.Position = 0; - stream.Write(header, 0, header.Length); + stream.Write(header); } /// @@ -98,24 +100,26 @@ internal static class SevenZipSignatureHeaderWriter CancellationToken cancellationToken = default ) { - var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc); + var header = new byte[HeaderSize]; + BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc); // Write at position 0 stream.Position = 0; await stream.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false); } - private static byte[] BuildFinalHeader( + private static void BuildFinalHeader( + Span header, ulong nextHeaderOffset, ulong nextHeaderSize, uint nextHeaderCrc ) { // Build StartHeader (20 bytes): NextHeaderOffset(8) + NextHeaderSize(8) + NextHeaderCRC(4) - var startHeader = new byte[20]; - BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(0, 8), nextHeaderOffset); - BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(8, 8), nextHeaderSize); - BinaryPrimitives.WriteUInt32LittleEndian(startHeader.AsSpan(16, 4), nextHeaderCrc); + var startHeader = header.Slice(12, 20); + BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(0, 8), nextHeaderOffset); + BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(8, 8), nextHeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(startHeader.Slice(16, 4), nextHeaderCrc); // CRC32 of StartHeader var startHeaderCrc = Crc32Stream.Compute( @@ -125,21 +129,14 @@ internal static class SevenZipSignatureHeaderWriter ); // Assemble full 32-byte header - var header = new byte[HeaderSize]; - // magic signature - Array.Copy(Signature, 0, header, 0, Signature.Length); + Signature.AsSpan().CopyTo(header); // version header[6] = 0; header[7] = 2; // StartHeaderCRC - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(8, 4), startHeaderCrc); - - // StartHeader - Array.Copy(startHeader, 0, header, 12, startHeader.Length); - - return header; + BinaryPrimitives.WriteUInt32LittleEndian(header.Slice(8, 4), startHeaderCrc); } } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs b/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs index 26b8692c..8754ed91 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs @@ -17,7 +17,7 @@ internal static class SevenZipWriteExtensions /// public static int WriteEncodedUInt64(this Stream stream, ulong value) { - var data = new byte[9]; + Span data = stackalloc byte[9]; data[0] = 0xFF; byte mask = 0x80; var length = 1; @@ -36,7 +36,7 @@ internal static class SevenZipWriteExtensions mask >>= 1; } - stream.Write(data, 0, length); + stream.Write(data.Slice(0, length)); return length; } diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 2790b7c5..5d4e3510 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -10,7 +10,7 @@ namespace SharpCompress.Compressors.LZMA; /// /// Write-only stream that compresses data using the LZMA2 framing format. /// Buffers input, compresses in chunks, and writes LZMA2-framed output to the underlying stream. -/// Each chunk is independently compressed with a fresh LZMA encoder. +/// Each chunk is independently compressed with reset LZMA state and reusable workspaces. /// internal sealed class Lzma2EncoderStream : Stream { @@ -30,6 +30,7 @@ internal sealed class Lzma2EncoderStream : Stream private readonly ArrayPool _bufferPool; private readonly byte[] _properties; private readonly Encoder _encoder; + private Decoder? _trialDecoder; private int _bufferPosition; private bool _isFirstChunk = true; private bool _isDisposed; @@ -239,6 +240,7 @@ internal sealed class Lzma2EncoderStream : Stream // Returning the input buffer must not depend on successful output or encoder cleanup. try { + _trialDecoder?.Dispose(); _encoder.Dispose(); } finally @@ -259,7 +261,7 @@ internal sealed class Lzma2EncoderStream : Stream _bufferPosition = 0; // Try compressing the data - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -271,13 +273,14 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; // Check if compressed output fits in a single chunk and is actually smaller if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length ) { - WriteCompressedChunk(uncompressedData.Length, compressed.Buffer, compressed.Length); + WriteCompressedChunk(uncompressedData.Length, compressed, (int)compressed.Length); } else { @@ -296,7 +299,7 @@ internal sealed class Lzma2EncoderStream : Stream var uncompressedData = _buffer.AsMemory(0, uncompressedSize); _bufferPosition = 0; - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -308,6 +311,7 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length @@ -315,8 +319,8 @@ internal sealed class Lzma2EncoderStream : Stream { await WriteCompressedChunkAsync( uncompressedData.Length, - compressed.Buffer, - compressed.Length, + compressed, + (int)compressed.Length, cancellationToken ) .ConfigureAwait(false); @@ -328,23 +332,31 @@ internal sealed class Lzma2EncoderStream : Stream } } - private (byte[] Buffer, int Length) CompressBlock(byte[] data, int length) + private PooledMemoryStream CompressBlock(byte[] data, int length) { using var inputMs = new MemoryStream(data, 0, length, writable: false); - using var outputMs = new PooledMemoryStream(); - - _encoder.Code(inputMs, outputMs, length, -1, null); - - var fullCompressed = outputMs.ToArray(); - - // The LZMA range encoder flush writes trailing bytes the decoder doesn't consume. - // Trial-decode to find the exact byte count the decoder needs, so the LZMA2 - // chunk header reports a compressed size that matches what the decoder reads. - var consumed = FindConsumedBytes(fullCompressed, fullCompressed.Length, length); - return (fullCompressed, consumed); + var outputMs = new PooledMemoryStream(); + try + { + _encoder.Code(inputMs, outputMs, length, -1, null); + outputMs.Position = 0; + // Trial decoding excludes range-encoder flush bytes from the chunk size. + var consumed = FindConsumedBytes(outputMs, (int)outputMs.Length, length); + outputMs.SetLength(consumed); + outputMs.Position = 0; + return outputMs; + } + catch + { + outputMs.Dispose(); + // A failed trial must not carry dictionary/output state into the next block. + _trialDecoder?.Dispose(); + _trialDecoder = null; + throw; + } } - private int FindConsumedBytes(byte[] compressedData, int compressedSize, int uncompressedSize) + private int FindConsumedBytes(Stream input, int compressedSize, int uncompressedSize) { // Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32] Span props = stackalloc byte[5]; @@ -354,10 +366,9 @@ internal sealed class Lzma2EncoderStream : Stream props[3] = (byte)(_dictionarySize >> 16); props[4] = (byte)(_dictionarySize >> 24); - using var decoder = new Decoder(); + var decoder = _trialDecoder ??= new Decoder(); decoder.SetDecoderProperties(props); - using var input = new MemoryStream(compressedData, 0, compressedSize, writable: false); decoder.Code(input, Stream.Null, compressedSize, uncompressedSize, null); return (int)input.Position; @@ -369,14 +380,14 @@ internal sealed class Lzma2EncoderStream : Stream /// private void WriteCompressedChunk( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize ) { var uncompSizeMinus1 = uncompressedSize - 1; var compSizeMinus1 = compressedSize - 1; - // Each chunk is compressed independently with a fresh LZMA encoder, + // Each chunk is compressed independently with reset LZMA encoder state, // so we must use 0xE0 (full reset: dictionary + state + properties) every time. // The decoder uses outWindow.Total for literal context and posState; // 0xE0 triggers outWindow.Reset() which zeros Total, matching the encoder's @@ -393,12 +404,12 @@ internal sealed class Lzma2EncoderStream : Stream // 0xE0 (>= 0xC0) requires properties byte _output.WriteByte(_lzmaPropertiesByte); - _output.Write(compressedData, 0, compressedSize); + compressedData.WriteTo(_output); } private async ValueTask WriteCompressedChunkAsync( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize, CancellationToken cancellationToken = default ) @@ -423,9 +434,7 @@ internal sealed class Lzma2EncoderStream : Stream { ArrayPool.Shared.Return(header); } - await _output - .WriteAsync(compressedData, 0, compressedSize, cancellationToken) - .ConfigureAwait(false); + await compressedData.CopyToAsync(_output, 65536, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs index 1d7ff15d..20315f6b 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs @@ -13,10 +13,17 @@ public partial class Decoder : IAsyncDisposable { public async ValueTask DisposeAsync() { - if (_outWindow is not null) + try + { + if (_outWindow is not null) + { + await _outWindow.DisposeAsync().ConfigureAwait(false); + } + } + finally { - await _outWindow.DisposeAsync().ConfigureAwait(false); _outWindow = null; + ReleaseModels(); } } @@ -157,16 +164,20 @@ public partial class Decoder : IAsyncDisposable } var rangeDecoder = new RangeCoder.Decoder(); - await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false); - - await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken) - .ConfigureAwait(false); - - await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false); - rangeDecoder.ReleaseStream(); - - await _outWindow.DisposeAsync().ConfigureAwait(false); - _outWindow = null; + try + { + await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false); + await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken) + .ConfigureAwait(false); + await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + rangeDecoder.ReleaseStream(); + var window = _outWindow; + _outWindow = null; + await window.DisposeAsync().ConfigureAwait(false); + } } internal async ValueTask CodeAsync( diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs index b25487ef..0b6339e3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs @@ -2,6 +2,7 @@ #if !LEGACY_DOTNET using System; +using System.Buffers; using System.Runtime.CompilerServices; using SharpCompress.Compressors.LZMA.LZ; @@ -46,17 +47,22 @@ public partial class Decoder private const int LenHighSize = 1 << Base.K_NUM_HIGH_LEN_BITS; // 256 private const int LenProbsSize = LenHighBase + LenHighSize; - private ushort[] _fIsMatch; - private ushort[] _fIsRep; - private ushort[] _fIsRepG0; - private ushort[] _fIsRepG1; - private ushort[] _fIsRepG2; - private ushort[] _fIsRep0Long; - private ushort[] _fPosSlot; - private ushort[] _fPosDecoders; - private ushort[] _fPosAlign; - private ushort[] _fLenProbs; - private ushort[] _fRepLenProbs; + private const int IsRepOffset = (int)Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX; + private const int IsRepG0Offset = IsRepOffset + (int)Base.K_NUM_STATES; + private const int IsRepG1Offset = IsRepG0Offset + (int)Base.K_NUM_STATES; + private const int IsRepG2Offset = IsRepG1Offset + (int)Base.K_NUM_STATES; + private const int IsRep0LongOffset = IsRepG2Offset + (int)Base.K_NUM_STATES; + private const int PosSlotOffset = IsRep0LongOffset + IsRepOffset; + private const int PosDecodersOffset = + PosSlotOffset + ((int)Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS); + private const int PosAlignOffset = + PosDecodersOffset + (int)(Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX); + private const int LenProbsOffset = PosAlignOffset + (1 << Base.K_NUM_ALIGN_BITS); + private const int RepLenProbsOffset = LenProbsOffset + LenProbsSize; + private const int LiteralOffset = RepLenProbsOffset + LenProbsSize; + + // Fixed probabilities precede literals in one rental. For default properties the + // whole bank fits a 16 KiB bucket, avoiding a separate rounded-up literal rental. private ushort[] _fLiteral; private int _fLiteralNumPrevBits = -1; @@ -65,43 +71,34 @@ public partial class Decoder private void CreateFastModel(int lp, int lc) { - _fIsMatch ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; - _fIsRep ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG0 ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG1 ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG2 ??= new ushort[Base.K_NUM_STATES]; - _fIsRep0Long ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; - _fPosSlot ??= new ushort[Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS]; - _fPosDecoders ??= new ushort[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX]; - _fPosAlign ??= new ushort[1 << Base.K_NUM_ALIGN_BITS]; - _fLenProbs ??= new ushort[LenProbsSize]; - _fRepLenProbs ??= new ushort[LenProbsSize]; - if (_fLiteralNumPrevBits != lc || _fLiteralNumPosBits != lp) { + var numStates = (uint)1 << (lc + lp); + var required = checked(LiteralOffset + (int)(numStates * 0x300)); + if (_fLiteral is null || _fLiteral.Length < required || _fLiteral.Length / 2 > required) + { + var replacement = ArrayPool.Shared.Rent(required); + if (_fLiteral is not null) + { + ArrayPool.Shared.Return(_fLiteral); + } + _fLiteral = replacement; + } _fLiteralNumPrevBits = lc; _fLiteralNumPosBits = lp; _fLiteralPosMask = ((uint)1 << lp) - 1; - var numStates = (uint)1 << (lc + lp); - _fLiteral = new ushort[checked((int)(numStates * 0x300))]; } } private void InitFastModel() { const ushort probInit = (ushort)(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL >> 1); - Array.Fill(_fIsMatch, probInit); - Array.Fill(_fIsRep, probInit); - Array.Fill(_fIsRepG0, probInit); - Array.Fill(_fIsRepG1, probInit); - Array.Fill(_fIsRepG2, probInit); - Array.Fill(_fIsRep0Long, probInit); - Array.Fill(_fPosSlot, probInit); - Array.Fill(_fPosDecoders, probInit); - Array.Fill(_fPosAlign, probInit); - Array.Fill(_fLenProbs, probInit); - Array.Fill(_fRepLenProbs, probInit); - Array.Fill(_fLiteral, probInit); + Array.Fill( + _fLiteral, + probInit, + 0, + LiteralOffset + (0x300 << (_fLiteralNumPrevBits + _fLiteralNumPosBits)) + ); } // Mutable range-coder state threaded through the decode helpers below as a single `ref` @@ -269,21 +266,19 @@ public partial class Decoder fixed (byte* pDic = dic) fixed (byte* pIn = inBuf) - fixed ( - ushort* pIsMatch = _fIsMatch, - pIsRep = _fIsRep, - pIsRepG0 = _fIsRepG0, - pIsRepG1 = _fIsRepG1, - pIsRepG2 = _fIsRepG2, - pIsRep0Long = _fIsRep0Long, - pPosSlot = _fPosSlot, - pPosDecoders = _fPosDecoders, - pPosAlign = _fPosAlign, - pLenProbs = _fLenProbs, - pRepLenProbs = _fRepLenProbs, - pLiteral = _fLiteral - ) + fixed (ushort* pIsMatch = _fLiteral) { + var pIsRep = pIsMatch + IsRepOffset; + var pIsRepG0 = pIsMatch + IsRepG0Offset; + var pIsRepG1 = pIsMatch + IsRepG1Offset; + var pIsRepG2 = pIsMatch + IsRepG2Offset; + var pIsRep0Long = pIsMatch + IsRep0LongOffset; + var pPosSlot = pIsMatch + PosSlotOffset; + var pPosDecoders = pIsMatch + PosDecodersOffset; + var pPosAlign = pIsMatch + PosAlignOffset; + var pLenProbs = pIsMatch + LenProbsOffset; + var pRepLenProbs = pIsMatch + RepLenProbsOffset; + var pLiteral = pIsMatch + LiteralOffset; var rs = new FastRangeState { Range = rangeDecoder._range, diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index 8cb96a2f..4e1de7f8 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading.Tasks; @@ -15,8 +16,30 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable public void Dispose() { - _outWindow?.Dispose(); - _outWindow = null; + try + { + _outWindow?.Dispose(); + } + finally + { + _outWindow = null; + ReleaseModels(); + } + } + + private void ReleaseModels() + { + _literalDecoder.ReleaseModels(); + scalarModelsReady = false; +#if !LEGACY_DOTNET + if (_fLiteral is not null) + { + ArrayPool.Shared.Return(_fLiteral); + _fLiteral = null; + } + _fLiteralNumPrevBits = _fLiteralNumPosBits = -1; + fastModelsReady = false; +#endif } private partial class LenDecoder @@ -137,16 +160,31 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { return; } + var numStates = (uint)1 << (numPrevBits + numPosBits); + var required = checked((int)(numStates * 0x300)); + // Allocate the reference-bearing wrappers before returning any old rental. + var coders = + _coders is null || _coders.Length < numStates || _coders.Length / 2 > numStates + ? new Decoder2[numStates] + : _coders; + // Do not retain a large property bank after switching to small contexts. + if (_models is null || _models.Length < required || _models.Length / 2 > required) + { + var replacement = ArrayPool.Shared.Rent(required); + if (_models is not null) + { + ArrayPool.Shared.Return(_models); + } + _models = replacement; + } + for (uint i = 0; i < numStates; i++) + { + coders[i].Create(_models, checked((int)(i * 0x300))); + } + _coders = coders; _numPosBits = numPosBits; _posMask = ((uint)1 << numPosBits) - 1; _numPrevBits = numPrevBits; - var numStates = (uint)1 << (_numPrevBits + _numPosBits); - _models = new BitDecoder[checked((int)(numStates * 0x300))]; - _coders = new Decoder2[numStates]; - for (uint i = 0; i < numStates; i++) - { - _coders[i].Create(_models, checked((int)(i * 0x300))); - } } public void Init() @@ -158,6 +196,16 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable } } + public void ReleaseModels() + { + if (_models is not null) + { + ArrayPool.Shared.Return(_models); + } + _models = null; + _coders = null; + } + private uint GetState(uint pos, byte prevByte) => ((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits)); @@ -174,28 +222,17 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable private OutWindow _outWindow; - private readonly BitDecoder[] _isMatchDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; - private readonly BitDecoder[] _isRepDecoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRep0LongDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; - - private readonly BitTreeDecoder[] _posSlotDecoder = new BitTreeDecoder[ - Base.K_NUM_LEN_TO_POS_STATES - ]; - private readonly BitDecoder[] _posDecoders = new BitDecoder[ - Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX - ]; - - private BitTreeDecoder _posAlignDecoder = new(Base.K_NUM_ALIGN_BITS); - - private readonly LenDecoder _lenDecoder = new(); - private readonly LenDecoder _repLenDecoder = new(); + private BitDecoder[] _isMatchDecoders; + private BitDecoder[] _isRepDecoders; + private BitDecoder[] _isRepG0Decoders; + private BitDecoder[] _isRepG1Decoders; + private BitDecoder[] _isRepG2Decoders; + private BitDecoder[] _isRep0LongDecoders; + private BitTreeDecoder[] _posSlotDecoder; + private BitDecoder[] _posDecoders; + private BitTreeDecoder _posAlignDecoder; + private LenDecoder _lenDecoder; + private LenDecoder _repLenDecoder; private readonly LiteralDecoder _literalDecoder = new(); @@ -218,10 +255,6 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable public Decoder() { _dictionarySize = -1; - for (var i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++) - { - _posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS); - } } #if !LEGACY_DOTNET @@ -302,6 +335,27 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { return; } + if (_isMatchDecoders is null) + { + // A sync-only decoder never needs this scalar object graph. + _isMatchDecoders = new BitDecoder[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; + _isRepDecoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRep0LongDecoders = new BitDecoder[ + Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX + ]; + _posSlotDecoder = new BitTreeDecoder[Base.K_NUM_LEN_TO_POS_STATES]; + for (var i = 0; i < _posSlotDecoder.Length; i++) + { + _posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS); + } + _posDecoders = new BitDecoder[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX]; + _posAlignDecoder = new BitTreeDecoder(Base.K_NUM_ALIGN_BITS); + _lenDecoder = new LenDecoder(); + _repLenDecoder = new LenDecoder(); + } _literalDecoder.Create(literalPosBits, literalContextBits); _lenDecoder.Create(_posStateMask + 1); _repLenDecoder.Create(_posStateMask + 1); @@ -332,15 +386,19 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable } var rangeDecoder = new RangeCoder.Decoder(); - rangeDecoder.Init(inStream); - - Code(_dictionarySize, _outWindow, rangeDecoder); - - _outWindow.ReleaseStream(); - rangeDecoder.ReleaseStream(); - - _outWindow.Dispose(); - _outWindow = null; + try + { + rangeDecoder.Init(inStream); + Code(_dictionarySize, _outWindow, rangeDecoder); + _outWindow.ReleaseStream(); + } + finally + { + rangeDecoder.ReleaseStream(); + var window = _outWindow; + _outWindow = null; + window.Dispose(); + } } #if !LEGACY_DOTNET diff --git a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs index 451acafe..4c04e45e 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs @@ -491,7 +491,7 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert public bool IsShortRep() => (_backPrev == 0); } - private readonly Optimal[] _optimum = new Optimal[K_NUM_OPTS]; + private Optimal[] _optimum; private BinTree _matchFinder; private readonly RangeCoder.Encoder _rangeEncoder = new(); @@ -605,6 +605,11 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert public void Dispose() { + if (_optimum is not null) + { + ArrayPool.Shared.Return(_optimum); + _optimum = null; + } _literalEncoder.Dispose(); _matchFinder?.Dispose(); _matchFinder = null; @@ -614,6 +619,9 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert private void Init() { + // This large workspace belongs to the encoder, not each input block. + _optimum ??= ArrayPool.Shared.Rent((int)K_NUM_OPTS); + Array.Clear(_optimum, 0, (int)K_NUM_OPTS); BaseInit(); _rangeEncoder.Init(); diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 1a0bf424..50906d03 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -465,8 +465,9 @@ public partial class LzmaStream cancellationToken.ThrowIfCancellationRequested(); if (_encoder != null) { + using var input = new SharpCompress.IO.ReadOnlyMemoryStream(buffer); _position = await _encoder - .CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken) + .CodeAsync(input, false, cancellationToken) .ConfigureAwait(false); } } @@ -523,7 +524,17 @@ public partial class LzmaStream try { #pragma warning disable VSTHRD103 // Encoder.Dispose only returns buffers; finalization is separate. - _encoder?.Dispose(); + try + { + _encoder?.Dispose(); + } + finally + { + if (_decoder is not null) + { + await _decoder.DisposeAsync().ConfigureAwait(false); + } + } #pragma warning restore VSTHRD103 } finally diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 0bae5c38..f068ae0d 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -267,7 +267,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { try { - _encoder?.Dispose(); + try + { + _encoder?.Dispose(); + } + finally + { + _decoder?.Dispose(); + } } finally { diff --git a/src/SharpCompress/IO/ReadOnlyMemoryStream.cs b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs new file mode 100644 index 00000000..bf5024d7 --- /dev/null +++ b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +// Borrows caller memory only until the awaited encoder operation completes. +internal sealed class ReadOnlyMemoryStream(ReadOnlyMemory memory) : Stream +{ + private int position; + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => memory.Length; + public override long Position + { + get => position; + set => throw new NotSupportedException(); + } + + private int ReadMemory(Span buffer) + { + var count = Math.Min(buffer.Length, memory.Length - position); + memory.Span.Slice(position, count).CopyTo(buffer); + position += count; + return count; + } + + public override int Read(byte[] buffer, int offset, int count) => + ReadMemory(buffer.AsSpan(offset, count)); + + public override int ReadByte() => position < memory.Length ? memory.Span[position++] : -1; + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Read(buffer, offset, count)); + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) => ReadMemory(buffer); + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask(ReadMemory(buffer.Span)); + } +#endif + + public override void Flush() { } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); +} diff --git a/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs b/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs new file mode 100644 index 00000000..6c7ffb0a --- /dev/null +++ b/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs @@ -0,0 +1,384 @@ +using System; +using System.Buffers; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common.SevenZip; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.LZMA.Utilities; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class LzmaScratchTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Raw_Decode_Failure_Releases_The_Dictionary(bool useAsync) + { + using var decoder = new SharpCompress.Compressors.LZMA.Decoder(); + decoder.SetDecoderProperties(new byte[] { 0x5D, 0, 0, 1, 0 }); + using var input = new FailingInput(); + if (useAsync) + { + await Assert.ThrowsAsync(async () => + await decoder.CodeAsync(input, Stream.Null, 32, 4096, null!, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => decoder.Code(input, Stream.Null, 32, 4096, null!)); + } + Assert.Null( + decoder + .GetType() + .GetField("_outWindow", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder) + ); + } + + private sealed class FailingInput : MemoryStream + { + public override int ReadByte() => throw new IOException("Read failure"); + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) => throw new IOException("Read failure"); + +#if !LEGACY_DOTNET + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) => throw new IOException("Read failure"); +#endif + } + + [Theory] + [InlineData(false)] +#if !LEGACY_DOTNET + [InlineData(true)] +#endif + public void Smaller_Literal_Properties_Release_Oversized_Banks(bool fast) + { + using var decoder = new SharpCompress.Compressors.LZMA.Decoder(); + object target = decoder; + var methodName = "CreateFastModel"; + var fieldName = "_fLiteral"; + if (!fast) + { + target = decoder + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder)!; + methodName = "Create"; + fieldName = "_models"; + } + var method = target + .GetType() + .GetMethod( + methodName, + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance + )!; + var field = target + .GetType() + .GetField(fieldName, BindingFlags.NonPublic | BindingFlags.Instance)!; + method.Invoke(target, new object[] { 0, 8 }); + var large = (Array)field.GetValue(target)!; + method.Invoke(target, new object[] { 0, 0 }); + var small = (Array)field.GetValue(target)!; + Assert.True(small.Length < large.Length); + Assert.InRange(small.Length, 768, 4096); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(7)] + [InlineData(8)] + [InlineData(9)] + [InlineData(17)] + public void Digests_Keep_Partial_Vector_And_Little_Endian_Encoding(int count) + { + var crcs = Enumerable + .Range(0, count) + .Select(i => i % 3 == 0 ? (uint?)(0x12345678u + (uint)i) : null) + .ToArray(); + using var expected = new MemoryStream(); + expected.WriteOptionalBoolVector(crcs.Select(c => c.HasValue).ToArray()); + foreach (var crc in crcs) + { + if (crc is { } value) + { + var bytes = new byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + expected.Write(bytes, 0, bytes.Length); + } + } + using var actual = new MemoryStream(); + new SevenZipDigestsWriter(crcs).Write(actual); + Assert.Equal(expected.ToArray(), actual.ToArray()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Partial_Decode_Disposal_Releases_Literal_Models(bool useAsync) + { + using var packed = new MemoryStream(); + var encoder = new Lzma2EncoderStream(packed, 1 << 16, 32); + var properties = encoder.Properties; + using (encoder) + { + encoder.Write(new byte[8192], 0, 8192); + } + packed.Position = 0; + using var stream = LzmaStream.Create( + properties, + packed, + packed.Length, + 8192, + leaveOpen: true + ); + var bytes = new byte[1]; + if (useAsync) + { + await stream.ReadAsync(bytes, 0, 1, CancellationToken.None); + } + else + { + stream.Read(bytes, 0, 1); + } + var decoder = stream + .GetType() + .GetField("_decoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(stream)!; + if (useAsync) + { + await stream.DisposeAsync(); + } + else + { + stream.Dispose(); + } + stream.Dispose(); + var literal = decoder + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder)!; + Assert.Null( + literal + .GetType() + .GetField("_models", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(literal) + ); +#if !LEGACY_DOTNET + Assert.Null( + decoder + .GetType() + .GetField("_fLiteral", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder) + ); +#endif + Assert.True(packed.CanRead); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Lzma2_Multiple_Blocks_RoundTrip(bool useAsync, bool random) + { + var expected = new byte[4 * 1024 * 1024 + 17]; + if (random) + { + new Random(987).NextBytes(expected); + } + else + { + for (var i = 0; i < expected.Length; i++) + { + expected[i] = (byte)(i % 17); + } + } + using var packed = new MemoryStream(); + byte[] properties; + var encoder = new Lzma2EncoderStream(packed, 1 << 16, 32); + properties = encoder.Properties; + if (useAsync) + { + await using (encoder) + { + await encoder.WriteAsync(expected, 0, expected.Length, CancellationToken.None); + } + } + else + { + using (encoder) + { + encoder.Write(expected, 0, expected.Length); + } + } + var trial = encoder + .GetType() + .GetField("_trialDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(encoder)!; + var literals = trial + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(trial)!; + Assert.Null( + literals + .GetType() + .GetField("_models", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(literals) + ); +#if !LEGACY_DOTNET + Assert.Null( + trial + .GetType() + .GetField("_fLiteral", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(trial) + ); +#endif + packed.Position = 0; + using var decoder = LzmaStream.Create( + properties, + packed, + packed.Length, + expected.Length, + leaveOpen: true + ); + using var actual = new MemoryStream(); + if (useAsync) + { + await decoder.CopyToAsync(actual, 65536, CancellationToken.None); + } + else + { + decoder.CopyTo(actual); + } + Assert.Equal(expected, actual.ToArray()); + Assert.True(packed.CanRead); + } + +#if !LEGACY_DOTNET + [Fact] + public async Task Lzma_Memory_Write_Borrows_Only_The_Requested_Slice() + { + var bytes = new byte[8192]; + new Random(654).NextBytes(bytes); + var expected = Enumerable.Range(0, 4).SelectMany(_ => bytes.Skip(7).Take(4096)).ToArray(); + using var memory = new NonArrayMemory(bytes); + using var packed = new MemoryStream(); + var encoder = LzmaStream.Create(new LzmaEncoderProperties(true, 1 << 16), false, packed); + var properties = encoder.Properties; + await using (encoder) + { + for (var i = 0; i < 4; i++) + { + await encoder.WriteAsync(memory.Memory.Slice(7, 4096), CancellationToken.None); + } + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + await Assert.ThrowsAnyAsync(async () => + await encoder.WriteAsync(bytes.AsMemory(), cancelled.Token) + ); + // Caller memory may be reused as soon as the awaited write completes, + // even though encoder finalization still has pending output. + Array.Clear(bytes, 0, bytes.Length); + } + packed.Position = 0; + using var decoder = LzmaStream.Create( + properties, + packed, + packed.Length, + 16384, + leaveOpen: true + ); + using var actual = new MemoryStream(); + await decoder.CopyToAsync(actual, 65536, CancellationToken.None); + Assert.Equal(expected, actual.ToArray()); + } +#endif + + private sealed class NonArrayMemory(byte[] bytes) : MemoryManager + { + public override Span GetSpan() => bytes; + + public override MemoryHandle Pin(int elementIndex = 0) => throw new NotSupportedException(); + + public override void Unpin() { } + + protected override void Dispose(bool disposing) { } + } + + [Fact] + public void Encoder_Workspace_Is_Lazy_And_Returned_On_Dispose() + { + using var encoder = new SharpCompress.Compressors.LZMA.Encoder(); + var field = encoder + .GetType() + .GetField("_optimum", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(encoder)); + using var input = new MemoryStream(new byte[4096]); + encoder.Code(input, Stream.Null, input.Length, -1, null!); + Assert.NotNull(field.GetValue(encoder)); + encoder.Dispose(); + encoder.Dispose(); + Assert.Null(field.GetValue(encoder)); + } + + [Fact] + public void FilesInfo_Streams_Names_And_Partial_Metadata() + { + var entries = Enumerable + .Range(0, 17) + .Select(i => new SevenZipWriteEntry + { + Name = i + "-\ud83d\ude00-\ud800-" + new string('x', i % 2 == 0 ? 300 : 5), + IsEmpty = true, + IsDirectory = i % 3 == 0, + ModificationTime = + i % 2 == 0 ? new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc) : null, + Attributes = i % 2 == 0 ? 32u : null, + }) + .ToArray(); + using var stream = new MemoryStream(); + new SevenZipFilesInfoWriter { Entries = entries }.Write(stream); + var bytes = stream.ToArray(); + var reader = new DataReader(bytes, 0, bytes.Length); + Assert.Equal(17, reader.ReadNum()); + while (true) + { + var property = (BlockType)reader.ReadByte(); + if (property == BlockType.End) + { + break; + } + var payload = new byte[reader.ReadNum()]; + reader.ReadBytes(payload, 0, payload.Length); + if (property == BlockType.Name) + { + Assert.Equal(0, payload[0]); + Assert.Equal( + Encoding.Unicode.GetBytes(string.Concat(entries.Select(e => e.Name + "\0"))), + payload.Skip(1) + ); + } + else if (property == BlockType.MTime || property == BlockType.WinAttributes) + { + Assert.Equal(new byte[] { 0, 0xAA, 0xAA, 0x80, 0 }, payload.Take(5)); + Assert.Equal(5 + 9 * (property == BlockType.MTime ? 8 : 4), payload.Length); + } + } + Assert.Equal(bytes.Length, reader.Offset); + } +}