From f41cc208cc9c08bb2dfce35ec261116c7a718ece Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 20 Oct 2026 08:28:23 +0100 Subject: [PATCH] Pool LZMA workspaces and streamline 7z metadata writing Reuse LZMA2 trial decoders and pooled compressed output, pool and consolidate decoder probability banks, lazily rent encoder optimization scratch, and avoid copying direct async memory writes. Use bounded stack scratch and stream 7z metadata without temporary property buffers. Add ownership, failure, context-shrink, Unicode, and block-boundary regressions. Document allocation profiles, the small-file LZMA2 throughput tradeoff, and rejection of exact-size match-finder arrays. Validation: 3,016 .NET 10 tests passed; all library targets, net48 tests, and performance project compile; formatting checks pass. --- docs/SEVENZIP_PROFILING.md | 137 +++++++ src/SharpCompress/Common/SevenZip/CFolder.cs | 3 +- .../Common/SevenZip/SevenZipFilesInfo.cs | 232 ++++++----- .../SevenZip/SevenZipHeaderStructures.cs | 38 +- .../SevenZip/SevenZipSignatureHeader.cs | 37 +- .../SevenZip/SevenZipWriteExtensions.cs | 4 +- .../Compressors/LZMA/Lzma2EncoderStream.cs | 65 +-- .../Compressors/LZMA/LzmaDecoder.Async.cs | 35 +- .../Compressors/LZMA/LzmaDecoder.Fast.cs | 97 +++-- .../Compressors/LZMA/LzmaDecoder.cs | 146 +++++-- .../Compressors/LZMA/LzmaEncoder.cs | 10 +- .../Compressors/LZMA/LzmaStream.Async.cs | 15 +- .../Compressors/LZMA/LzmaStream.cs | 9 +- src/SharpCompress/IO/ReadOnlyMemoryStream.cs | 67 +++ .../Streams/LzmaScratchTests.cs | 384 ++++++++++++++++++ 15 files changed, 1014 insertions(+), 265 deletions(-) create mode 100644 src/SharpCompress/IO/ReadOnlyMemoryStream.cs create mode 100644 tests/SharpCompress.Test/Streams/LzmaScratchTests.cs diff --git a/docs/SEVENZIP_PROFILING.md b/docs/SEVENZIP_PROFILING.md index f4be5aa6..80c07838 100644 --- a/docs/SEVENZIP_PROFILING.md +++ b/docs/SEVENZIP_PROFILING.md @@ -314,3 +314,140 @@ Validation for this follow-up: **466 targeted tests** and **2,997 complete .NET succeed; net48 tests were not run on macOS. Formatting and whitespace checks pass. No public signatures or CRC-validation semantics changed. This phase did not introduce a global pool or increase the production dictionary-growth factor. + +## Follow-up: encoder workspaces and metadata scratch + +This round starts at `03056961`, after the preceding changes. Saved libraries, +benchmark sources, JSON logs and EventPipe traces live in the external artifact +directory's `round3/` subdirectory. Comparisons use Release net10.0, two warmups +and five measured microbenchmark iterations. Large extraction uses the existing +200 MiB solid LZMA2 archive, two warmups and three measured iterations. Inputs, +metadata objects and reflection delegates are prepared outside measurement. +Builds and tests do not run concurrently with measured workloads. + +### Implementation + +- The encoder rents its 4,096-element optimization workspace on first use, + clears its logical range on initialization and returns it on disposal. This + removes a recurring large-object allocation without pooling encoder objects. +- LZMA2 block compression keeps its pooled output stream alive through trial + decoding and chunk writing instead of materializing it with `ToArray()`. + The output is truncated to the trial decoder's consumed byte count, preserving + range-coder flush-byte handling. Uncompressed fallback still works; exceptions + return the temporary output and invalidate failed trial-decoder state. +- A trial decoder is reused within each LZMA2 encoder stream. Fixed scalar + decoder tables initialize only when scalar decoding is actually needed. + Scalar literal probabilities are pooled. Fast probabilities share one pooled + bank containing fixed tables followed by literals, keeping the default bank + within a 16 KiB pool bucket. Initialization uses logical lengths, not capacity. + Both stream-disposal paths now dispose their decoder to return model rentals. +- Direct `LzmaStream.WriteAsync(ReadOnlyMemory)` borrows memory through an + internal read-only adapter until encoding completes, rather than copying it. + This includes memory not backed by an array. The current 7z copy loop calls the + array overload, so this benefit is measured separately from archive writing. +- Synchronous integer, digest, coder-ID, timestamp, attribute and signature-header + scratch uses bounded stack buffers. Folder validation's 32-word stack mask is + explicitly cleared. Async signature headers retain owned storage across awaits. + Legacy span-write compatibility can still rent/copy; it is not a zero-copy claim. +- FilesInfo computes property sizes and writes directly, eliminating temporary + property streams, closures and bool arrays. Names use 512-byte stack scratch + for short names on modern targets and a reusable rented buffer for long names. + Legacy encoding uses a rented buffer. CRC-defined vectors also write directly. + +No public API signatures, dictionary-growth policy, stream ownership defaults +or checksum-validation semantics change. + +### Match-finder bucket experiment rejected + +The binary-tree match finder needs `2 * (dictionarySize + 1)` uint entries. +For a 1 MiB dictionary, its logical payload is 8 MiB + 8 bytes, crossing into a +16 MiB shared-pool bucket on modern .NET. An experimental build replaced this +default rental with an exact-size owned array, retaining custom-pool behavior. +It reduced capacity per active encoder but lost warm buffer reuse: 16 small +files allocated **145,934,592 B for LZMA** and **143,684,800 B for LZMA2**, with +median elapsed times of 20.7 ms and 26.7 ms respectively. This is substantially +worse allocation behavior than the retained shared-pool implementation. + +The exact-array experiment was removed. A bounded exact-size reusable cache +could address both concerns, but was not introduced: it requires a separate +retention/concurrency policy rather than silently replacing shared pooling with +an unbounded second cache or reducing the effective dictionary size. Likewise, +the 17 KiB length-price tables are not independently pooled into 32 KiB buckets; +this round prioritizes the existing large optimization workspace and model bank. + +### Results + +Exact managed allocation deltas are medians after warmup. These results describe +pool reuse, not guaranteed allocation-free cold starts or lower process RSS. +Pool capacity can exceed logical data sizes; oversized literal banks are released +when switching to substantially smaller context properties rather than retained +for the decoder's entire remaining lifetime. + +| Workload | Baseline allocated | Updated allocated | +| --- | ---: | ---: | +| 16 independent 4 KiB files, LZMA writer sync | 6,389,192 B | 3,319,696 B | +| Same files, LZMA writer async | 4,338,864 B | 1,269,480 B | +| Same files, LZMA2 writer sync | 4,582,864 B | 1,071,896 B | +| Same files, LZMA2 writer async | 4,658,968 B | 1,148,112 B | +| Three compressible 2 MiB LZMA2 blocks, sync/async | 307,360 B | 55,144 B | +| Three random 2 MiB LZMA2 blocks, sync/async | 6,684,832 B | 56,704 B | +| Serialize 10,000 mixed metadata entries | 2,868,992 B | 40 B | +| 64 direct 4 KiB LZMA memory writes | 501,360 B | 56,408 B | +| Decode 64 independent reset chunks, sync | 24,072 B | 712 B | +| Decode 64 independent reset chunks, async | 33,472 B | 8,784 B | + +The metadata microbenchmark includes long/short Unicode names, partial times, +directories and empty files. Metadata objects and delegates already exist; it +does not include archive creation, header compression, or public entry objects. +The remaining 40 bytes are the harness stopwatch. The block-compression benchmark +discards output to `Stream.Null`; round-trip tests independently validate emitted +data. The writer benchmark includes finalization and seekable MemoryStream output. + +| 200 MiB extraction | Baseline allocated | Updated allocated | Baseline elapsed | Updated elapsed | +| --- | ---: | ---: | ---: | ---: | +| Archive entries sync | 57,640 B | 9,992 B | 2.612 s | 2.554 s | +| Archive entries async | 76,024 B | 26,344 B | 14.420 s | 14.130 s | + +Allocation savings are approximately **83% sync / 65% async** relative to this +round's already-optimized baseline. These raw entry-stream extractions retain +the earlier report's checksum-validation caveat. + +Writer elapsed times were noisy in the main run. A separate three-process-pair +repeat alternated baseline/final execution order; each process warmed twice and +measured five iterations. Medians across the 15 measured iterations per version: + +| Small-file writer | Baseline elapsed | Updated elapsed | +| --- | ---: | ---: | +| LZMA sync | 18.57 ms | 18.66 ms | +| LZMA async | 34.49 ms | 34.87 ms | +| LZMA2 sync | 22.51 ms | 23.52 ms | +| LZMA2 async | 22.36 ms | 23.15 ms | + +LZMA is essentially unchanged; **small-file LZMA2 costs about 3–5% elapsed time** +in this repeat while allocating 75–77% less. The allocation-focused changes are +retained with that tradeoff explicit, not described as a universal throughput +improvement. Large/block timing variation likewise does not justify a general +speedup claim. + +Separate baseline/final EventPipe sampled-thread-time and allocation traces of +random-block writing report **zero lost events**. Baseline measured-phase samples +include about 31.9 MB of byte arrays and 0.90 MB of `Optimal[]`; the final trace +has no corresponding recurring samples. Remaining samples are sparse. CPU leaves +remain dominated by encoder block processing, binary-tree matching and trial +decoding. Exact allocation deltas are stronger evidence than sampled percentages. + +`run.py` reproduces microbenchmarks, extraction comparisons and traces; +`repeat.py` reproduces the interleaved writer timing check; `exact.py` measures +the rejected exact-array match-finder snapshot. Baseline, intermediate, +consolidated-final and exact-array libraries are saved separately. + +### Validation + +- **485 targeted tests** and **3,016 complete .NET 10 Release tests** pass. +- All six library targets, net48 tests, and the performance project compile + without warnings/errors. net48 tests are not executed on macOS. +- New coverage exercises multiple compressed/uncompressed LZMA2 blocks, partial + decoder disposal, failed raw decoding, pooled workspace disposal, shrinking + context banks, non-array memory writes, cancellation and caller-memory reuse, + long/invalid-surrogate names, partial metadata and digest-vector byte boundaries. +- CSharpier formatting/check and whitespace validation pass. diff --git a/src/SharpCompress/Common/SevenZip/CFolder.cs b/src/SharpCompress/Common/SevenZip/CFolder.cs index 683c27ca..eb95da39 100644 --- a/src/SharpCompress/Common/SevenZip/CFolder.cs +++ b/src/SharpCompress/Common/SevenZip/CFolder.cs @@ -138,7 +138,8 @@ internal class CFolder } } - var mask = new uint[kMaskSize]; + Span mask = stackalloc uint[kMaskSize]; + mask.Clear(); { var inStreamToCoder = new List(); diff --git a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs index b06b13ea..10a832e4 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs @@ -1,8 +1,8 @@ using System; +using System.Buffers; using System.IO; using System.Text; using SharpCompress.Compressors.LZMA.Utilities; -using SharpCompress.IO; namespace SharpCompress.Common.SevenZip; @@ -63,18 +63,13 @@ internal sealed class SevenZipFilesInfoWriter private void WriteEmptyStreamProperty(Stream stream, int emptyStreamCount) { - var emptyStreams = new bool[Entries.Length]; - var emptyFiles = new bool[emptyStreamCount]; var hasEmptyFile = false; - var emptyIndex = 0; for (var i = 0; i < Entries.Length; i++) { if (Entries[i].IsEmpty || Entries[i].IsDirectory) { - emptyStreams[i] = true; var isEmptyFile = !Entries[i].IsDirectory; - emptyFiles[emptyIndex++] = isEmptyFile; if (isEmptyFile) { hasEmptyFile = true; @@ -83,147 +78,194 @@ internal sealed class SevenZipFilesInfoWriter } // kEmptyStream - WriteFileProperty(stream, BlockType.EmptyStream, s => s.WriteBoolVector(emptyStreams)); + WritePropertyHeader(stream, BlockType.EmptyStream, (Entries.LongLength + 7) / 8); + WriteFlags(stream, BlockType.EmptyStream); // kEmptyFile (only if there are actual empty files, not just directories) if (hasEmptyFile) { - WriteFileProperty(stream, BlockType.EmptyFile, s => s.WriteBoolVector(emptyFiles)); + WritePropertyHeader(stream, BlockType.EmptyFile, ((long)emptyStreamCount + 7) / 8); + WriteFlags(stream, BlockType.EmptyFile); } } private void WriteNameProperty(Stream stream) { - WriteFileProperty( - stream, - BlockType.Name, - s => + long size = 1; + var maxBytes = 0; + foreach (var entry in Entries) + { + var count = Encoding.Unicode.GetByteCount(entry.Name); + size = checked(size + count + 2); + maxBytes = Math.Max(maxBytes, count); + } + WritePropertyHeader(stream, BlockType.Name, size); + stream.WriteByte(0); + byte[]? rented = null; +#if !LEGACY_DOTNET + Span small = stackalloc byte[512]; +#endif + try + { + foreach (var entry in Entries) { - // External = 0 (inline) - s.WriteByte(0); - - for (var i = 0; i < Entries.Length; i++) +#if !LEGACY_DOTNET + var count = Encoding.Unicode.GetByteCount(entry.Name); + if (count <= small.Length) { - var nameBytes = Encoding.Unicode.GetBytes(Entries[i].Name); - s.Write(nameBytes); - // null terminator (2 bytes for UTF-16) - s.WriteByte(0); - s.WriteByte(0); + Encoding.Unicode.GetBytes(entry.Name.AsSpan(), small); + stream.Write(small.Slice(0, count)); } + else +#endif + { + rented ??= ArrayPool.Shared.Rent(maxBytes); + var written = Encoding.Unicode.GetBytes( + entry.Name, + 0, + entry.Name.Length, + rented, + 0 + ); + stream.Write(rented, 0, written); + } + stream.WriteByte(0); + stream.WriteByte(0); } - ); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } } private void WriteMTimeProperty(Stream stream) { - var hasTimes = false; + var definedCount = 0; for (var i = 0; i < Entries.Length; i++) { - if (Entries[i].ModificationTime != null) + if (Entries[i].ModificationTime is not null) { - hasTimes = true; - break; + definedCount++; } } - if (!hasTimes) + if (definedCount == 0) { return; } - WriteFileProperty( + WritePropertyHeader( stream, BlockType.MTime, - s => - { - var defined = new bool[Entries.Length]; - for (var i = 0; i < Entries.Length; i++) - { - defined[i] = Entries[i].ModificationTime != null; - } - s.WriteOptionalBoolVector(defined); - - // External = 0 (inline) - s.WriteByte(0); - - var buf = new byte[8]; - for (var i = 0; i < Entries.Length; i++) - { - if (Entries[i].ModificationTime is { } mtime) - { - var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc(); - System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian( - buf, - fileTime - ); - s.Write(buf, 0, 8); - } - } - } + 2L + + (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8) + + 8L * definedCount ); + stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0)); + if (definedCount != Entries.Length) + { + WriteFlags(stream, BlockType.MTime); + } + stream.WriteByte(0); + Span buf = stackalloc byte[8]; + for (var i = 0; i < Entries.Length; i++) + { + if (Entries[i].ModificationTime is { } mtime) + { + var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc(); + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(buf, fileTime); + stream.Write(buf); + } + } } private void WriteAttributesProperty(Stream stream) { - var hasAttrs = false; + var definedCount = 0; for (var i = 0; i < Entries.Length; i++) { - if (Entries[i].Attributes != null) + if (Entries[i].Attributes is not null) { - hasAttrs = true; - break; + definedCount++; } } - if (!hasAttrs) + if (definedCount == 0) { return; } - WriteFileProperty( + WritePropertyHeader( stream, BlockType.WinAttributes, - s => - { - var defined = new bool[Entries.Length]; - for (var i = 0; i < Entries.Length; i++) - { - defined[i] = Entries[i].Attributes != null; - } - s.WriteOptionalBoolVector(defined); - - // External = 0 (inline) - s.WriteByte(0); - - var buf = new byte[4]; - for (var i = 0; i < Entries.Length; i++) - { - if (Entries[i].Attributes is { } attrs) - { - System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs); - s.Write(buf, 0, 4); - } - } - } + 2L + + (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8) + + 4L * definedCount ); + stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0)); + if (definedCount != Entries.Length) + { + WriteFlags(stream, BlockType.WinAttributes); + } + stream.WriteByte(0); + Span buf = stackalloc byte[4]; + for (var i = 0; i < Entries.Length; i++) + { + if (Entries[i].Attributes is { } attrs) + { + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs); + stream.Write(buf); + } + } } /// - /// Writes a file property block: PropertyID + size + data. - /// Size is computed by writing to a temporary buffer first. + /// Writes a property prefix without staging its computable payload in another stream. /// - private static void WriteFileProperty( - Stream stream, - BlockType propertyId, - Action writeData - ) + private static void WritePropertyHeader(Stream stream, BlockType propertyId, long size) { - using var dataStream = new PooledMemoryStream(); - writeData(dataStream); - stream.WriteByte((byte)propertyId); - stream.WriteEncodedUInt64((ulong)dataStream.Length); - dataStream.Position = 0; - dataStream.CopyTo(stream); + stream.WriteEncodedUInt64((ulong)size); + } + + private void WriteFlags(Stream stream, BlockType property) + { + byte mask = 0x80; + byte bits = 0; + foreach (var entry in Entries) + { + var empty = entry.IsEmpty || entry.IsDirectory; + if (property == BlockType.EmptyFile && !empty) + { + continue; + } + var defined = property switch + { + BlockType.EmptyStream => empty, + BlockType.EmptyFile => !entry.IsDirectory, + BlockType.MTime => entry.ModificationTime is not null, + BlockType.WinAttributes => entry.Attributes is not null, + _ => throw new InvalidOperationException(), + }; + if (defined) + { + bits |= mask; + } + mask >>= 1; + if (mask == 0) + { + stream.WriteByte(bits); + mask = 0x80; + bits = 0; + } + } + if (mask != 0x80) + { + stream.WriteByte(bits); + } } } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs b/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs index 24e86bfd..bfbc4718 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs @@ -13,21 +13,43 @@ internal sealed class SevenZipDigestsWriter(uint?[] crcs) public void Write(Stream stream) { - var defined = new bool[CRCs.Length]; + var allDefined = true; for (var i = 0; i < CRCs.Length; i++) { - defined[i] = CRCs[i] != null; + allDefined &= CRCs[i] is not null; + } + stream.WriteByte((byte)(allDefined ? 1 : 0)); + if (!allDefined) + { + byte mask = 0x80; + byte bits = 0; + foreach (var crc in CRCs) + { + if (crc is not null) + { + bits |= mask; + } + mask >>= 1; + if (mask == 0) + { + stream.WriteByte(bits); + mask = 0x80; + bits = 0; + } + } + if (mask != 0x80) + { + stream.WriteByte(bits); + } } - stream.WriteOptionalBoolVector(defined); - - var buf = new byte[4]; + Span buf = stackalloc byte[4]; for (var i = 0; i < CRCs.Length; i++) { if (CRCs[i] is { } crcValue) { System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, crcValue); - stream.Write(buf, 0, 4); + stream.Write(buf); } } } @@ -184,14 +206,14 @@ internal sealed class SevenZipUnPackInfoWriter stream.WriteByte(attributes); // Codec ID bytes (big-endian, most significant byte first) - var codecId = new byte[codecIdLength]; + Span codecId = stackalloc byte[8]; var id = coder._methodId._id; for (var i = codecIdLength - 1; i >= 0; i--) { codecId[i] = (byte)(id & 0xFF); id >>= 8; } - stream.Write(codecId, 0, codecIdLength); + stream.Write(codecId.Slice(0, codecIdLength)); if (isComplex) { diff --git a/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs b/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs index 060bc654..3733cb15 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs @@ -29,17 +29,18 @@ internal static class SevenZipSignatureHeaderWriter /// public static void WritePlaceholder(Stream stream) { - var header = new byte[HeaderSize]; + Span header = stackalloc byte[HeaderSize]; + header.Clear(); // magic signature - Array.Copy(Signature, 0, header, 0, Signature.Length); + Signature.AsSpan().CopyTo(header); // version: major=0, minor=2 (standard 7z format) header[6] = 0; header[7] = 2; // remaining 24 bytes are zero (placeholder for CRC and StartHeader) - stream.Write(header, 0, header.Length); + stream.Write(header); } /// @@ -79,11 +80,12 @@ internal static class SevenZipSignatureHeaderWriter uint nextHeaderCrc ) { - var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc); + Span header = stackalloc byte[HeaderSize]; + BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc); // Write at position 0 stream.Position = 0; - stream.Write(header, 0, header.Length); + stream.Write(header); } /// @@ -98,24 +100,26 @@ internal static class SevenZipSignatureHeaderWriter CancellationToken cancellationToken = default ) { - var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc); + var header = new byte[HeaderSize]; + BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc); // Write at position 0 stream.Position = 0; await stream.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false); } - private static byte[] BuildFinalHeader( + private static void BuildFinalHeader( + Span header, ulong nextHeaderOffset, ulong nextHeaderSize, uint nextHeaderCrc ) { // Build StartHeader (20 bytes): NextHeaderOffset(8) + NextHeaderSize(8) + NextHeaderCRC(4) - var startHeader = new byte[20]; - BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(0, 8), nextHeaderOffset); - BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(8, 8), nextHeaderSize); - BinaryPrimitives.WriteUInt32LittleEndian(startHeader.AsSpan(16, 4), nextHeaderCrc); + var startHeader = header.Slice(12, 20); + BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(0, 8), nextHeaderOffset); + BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(8, 8), nextHeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(startHeader.Slice(16, 4), nextHeaderCrc); // CRC32 of StartHeader var startHeaderCrc = Crc32Stream.Compute( @@ -125,21 +129,14 @@ internal static class SevenZipSignatureHeaderWriter ); // Assemble full 32-byte header - var header = new byte[HeaderSize]; - // magic signature - Array.Copy(Signature, 0, header, 0, Signature.Length); + Signature.AsSpan().CopyTo(header); // version header[6] = 0; header[7] = 2; // StartHeaderCRC - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(8, 4), startHeaderCrc); - - // StartHeader - Array.Copy(startHeader, 0, header, 12, startHeader.Length); - - return header; + BinaryPrimitives.WriteUInt32LittleEndian(header.Slice(8, 4), startHeaderCrc); } } diff --git a/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs b/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs index 26b8692c..8754ed91 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs @@ -17,7 +17,7 @@ internal static class SevenZipWriteExtensions /// public static int WriteEncodedUInt64(this Stream stream, ulong value) { - var data = new byte[9]; + Span data = stackalloc byte[9]; data[0] = 0xFF; byte mask = 0x80; var length = 1; @@ -36,7 +36,7 @@ internal static class SevenZipWriteExtensions mask >>= 1; } - stream.Write(data, 0, length); + stream.Write(data.Slice(0, length)); return length; } diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 2790b7c5..5d4e3510 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -10,7 +10,7 @@ namespace SharpCompress.Compressors.LZMA; /// /// Write-only stream that compresses data using the LZMA2 framing format. /// Buffers input, compresses in chunks, and writes LZMA2-framed output to the underlying stream. -/// Each chunk is independently compressed with a fresh LZMA encoder. +/// Each chunk is independently compressed with reset LZMA state and reusable workspaces. /// internal sealed class Lzma2EncoderStream : Stream { @@ -30,6 +30,7 @@ internal sealed class Lzma2EncoderStream : Stream private readonly ArrayPool _bufferPool; private readonly byte[] _properties; private readonly Encoder _encoder; + private Decoder? _trialDecoder; private int _bufferPosition; private bool _isFirstChunk = true; private bool _isDisposed; @@ -239,6 +240,7 @@ internal sealed class Lzma2EncoderStream : Stream // Returning the input buffer must not depend on successful output or encoder cleanup. try { + _trialDecoder?.Dispose(); _encoder.Dispose(); } finally @@ -259,7 +261,7 @@ internal sealed class Lzma2EncoderStream : Stream _bufferPosition = 0; // Try compressing the data - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -271,13 +273,14 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; // Check if compressed output fits in a single chunk and is actually smaller if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length ) { - WriteCompressedChunk(uncompressedData.Length, compressed.Buffer, compressed.Length); + WriteCompressedChunk(uncompressedData.Length, compressed, (int)compressed.Length); } else { @@ -296,7 +299,7 @@ internal sealed class Lzma2EncoderStream : Stream var uncompressedData = _buffer.AsMemory(0, uncompressedSize); _bufferPosition = 0; - (byte[] Buffer, int Length) compressed; + PooledMemoryStream compressed; try { compressed = CompressBlock(_buffer, uncompressedSize); @@ -308,6 +311,7 @@ internal sealed class Lzma2EncoderStream : Stream return; } + using var compressedScope = compressed; if ( compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE && compressed.Length < uncompressedData.Length @@ -315,8 +319,8 @@ internal sealed class Lzma2EncoderStream : Stream { await WriteCompressedChunkAsync( uncompressedData.Length, - compressed.Buffer, - compressed.Length, + compressed, + (int)compressed.Length, cancellationToken ) .ConfigureAwait(false); @@ -328,23 +332,31 @@ internal sealed class Lzma2EncoderStream : Stream } } - private (byte[] Buffer, int Length) CompressBlock(byte[] data, int length) + private PooledMemoryStream CompressBlock(byte[] data, int length) { using var inputMs = new MemoryStream(data, 0, length, writable: false); - using var outputMs = new PooledMemoryStream(); - - _encoder.Code(inputMs, outputMs, length, -1, null); - - var fullCompressed = outputMs.ToArray(); - - // The LZMA range encoder flush writes trailing bytes the decoder doesn't consume. - // Trial-decode to find the exact byte count the decoder needs, so the LZMA2 - // chunk header reports a compressed size that matches what the decoder reads. - var consumed = FindConsumedBytes(fullCompressed, fullCompressed.Length, length); - return (fullCompressed, consumed); + var outputMs = new PooledMemoryStream(); + try + { + _encoder.Code(inputMs, outputMs, length, -1, null); + outputMs.Position = 0; + // Trial decoding excludes range-encoder flush bytes from the chunk size. + var consumed = FindConsumedBytes(outputMs, (int)outputMs.Length, length); + outputMs.SetLength(consumed); + outputMs.Position = 0; + return outputMs; + } + catch + { + outputMs.Dispose(); + // A failed trial must not carry dictionary/output state into the next block. + _trialDecoder?.Dispose(); + _trialDecoder = null; + throw; + } } - private int FindConsumedBytes(byte[] compressedData, int compressedSize, int uncompressedSize) + private int FindConsumedBytes(Stream input, int compressedSize, int uncompressedSize) { // Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32] Span props = stackalloc byte[5]; @@ -354,10 +366,9 @@ internal sealed class Lzma2EncoderStream : Stream props[3] = (byte)(_dictionarySize >> 16); props[4] = (byte)(_dictionarySize >> 24); - using var decoder = new Decoder(); + var decoder = _trialDecoder ??= new Decoder(); decoder.SetDecoderProperties(props); - using var input = new MemoryStream(compressedData, 0, compressedSize, writable: false); decoder.Code(input, Stream.Null, compressedSize, uncompressedSize, null); return (int)input.Position; @@ -369,14 +380,14 @@ internal sealed class Lzma2EncoderStream : Stream /// private void WriteCompressedChunk( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize ) { var uncompSizeMinus1 = uncompressedSize - 1; var compSizeMinus1 = compressedSize - 1; - // Each chunk is compressed independently with a fresh LZMA encoder, + // Each chunk is compressed independently with reset LZMA encoder state, // so we must use 0xE0 (full reset: dictionary + state + properties) every time. // The decoder uses outWindow.Total for literal context and posState; // 0xE0 triggers outWindow.Reset() which zeros Total, matching the encoder's @@ -393,12 +404,12 @@ internal sealed class Lzma2EncoderStream : Stream // 0xE0 (>= 0xC0) requires properties byte _output.WriteByte(_lzmaPropertiesByte); - _output.Write(compressedData, 0, compressedSize); + compressedData.WriteTo(_output); } private async ValueTask WriteCompressedChunkAsync( int uncompressedSize, - byte[] compressedData, + PooledMemoryStream compressedData, int compressedSize, CancellationToken cancellationToken = default ) @@ -423,9 +434,7 @@ internal sealed class Lzma2EncoderStream : Stream { ArrayPool.Shared.Return(header); } - await _output - .WriteAsync(compressedData, 0, compressedSize, cancellationToken) - .ConfigureAwait(false); + await compressedData.CopyToAsync(_output, 65536, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs index 1d7ff15d..20315f6b 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs @@ -13,10 +13,17 @@ public partial class Decoder : IAsyncDisposable { public async ValueTask DisposeAsync() { - if (_outWindow is not null) + try + { + if (_outWindow is not null) + { + await _outWindow.DisposeAsync().ConfigureAwait(false); + } + } + finally { - await _outWindow.DisposeAsync().ConfigureAwait(false); _outWindow = null; + ReleaseModels(); } } @@ -157,16 +164,20 @@ public partial class Decoder : IAsyncDisposable } var rangeDecoder = new RangeCoder.Decoder(); - await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false); - - await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken) - .ConfigureAwait(false); - - await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false); - rangeDecoder.ReleaseStream(); - - await _outWindow.DisposeAsync().ConfigureAwait(false); - _outWindow = null; + try + { + await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false); + await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken) + .ConfigureAwait(false); + await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + rangeDecoder.ReleaseStream(); + var window = _outWindow; + _outWindow = null; + await window.DisposeAsync().ConfigureAwait(false); + } } internal async ValueTask CodeAsync( diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs index b25487ef..0b6339e3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs @@ -2,6 +2,7 @@ #if !LEGACY_DOTNET using System; +using System.Buffers; using System.Runtime.CompilerServices; using SharpCompress.Compressors.LZMA.LZ; @@ -46,17 +47,22 @@ public partial class Decoder private const int LenHighSize = 1 << Base.K_NUM_HIGH_LEN_BITS; // 256 private const int LenProbsSize = LenHighBase + LenHighSize; - private ushort[] _fIsMatch; - private ushort[] _fIsRep; - private ushort[] _fIsRepG0; - private ushort[] _fIsRepG1; - private ushort[] _fIsRepG2; - private ushort[] _fIsRep0Long; - private ushort[] _fPosSlot; - private ushort[] _fPosDecoders; - private ushort[] _fPosAlign; - private ushort[] _fLenProbs; - private ushort[] _fRepLenProbs; + private const int IsRepOffset = (int)Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX; + private const int IsRepG0Offset = IsRepOffset + (int)Base.K_NUM_STATES; + private const int IsRepG1Offset = IsRepG0Offset + (int)Base.K_NUM_STATES; + private const int IsRepG2Offset = IsRepG1Offset + (int)Base.K_NUM_STATES; + private const int IsRep0LongOffset = IsRepG2Offset + (int)Base.K_NUM_STATES; + private const int PosSlotOffset = IsRep0LongOffset + IsRepOffset; + private const int PosDecodersOffset = + PosSlotOffset + ((int)Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS); + private const int PosAlignOffset = + PosDecodersOffset + (int)(Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX); + private const int LenProbsOffset = PosAlignOffset + (1 << Base.K_NUM_ALIGN_BITS); + private const int RepLenProbsOffset = LenProbsOffset + LenProbsSize; + private const int LiteralOffset = RepLenProbsOffset + LenProbsSize; + + // Fixed probabilities precede literals in one rental. For default properties the + // whole bank fits a 16 KiB bucket, avoiding a separate rounded-up literal rental. private ushort[] _fLiteral; private int _fLiteralNumPrevBits = -1; @@ -65,43 +71,34 @@ public partial class Decoder private void CreateFastModel(int lp, int lc) { - _fIsMatch ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; - _fIsRep ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG0 ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG1 ??= new ushort[Base.K_NUM_STATES]; - _fIsRepG2 ??= new ushort[Base.K_NUM_STATES]; - _fIsRep0Long ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; - _fPosSlot ??= new ushort[Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS]; - _fPosDecoders ??= new ushort[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX]; - _fPosAlign ??= new ushort[1 << Base.K_NUM_ALIGN_BITS]; - _fLenProbs ??= new ushort[LenProbsSize]; - _fRepLenProbs ??= new ushort[LenProbsSize]; - if (_fLiteralNumPrevBits != lc || _fLiteralNumPosBits != lp) { + var numStates = (uint)1 << (lc + lp); + var required = checked(LiteralOffset + (int)(numStates * 0x300)); + if (_fLiteral is null || _fLiteral.Length < required || _fLiteral.Length / 2 > required) + { + var replacement = ArrayPool.Shared.Rent(required); + if (_fLiteral is not null) + { + ArrayPool.Shared.Return(_fLiteral); + } + _fLiteral = replacement; + } _fLiteralNumPrevBits = lc; _fLiteralNumPosBits = lp; _fLiteralPosMask = ((uint)1 << lp) - 1; - var numStates = (uint)1 << (lc + lp); - _fLiteral = new ushort[checked((int)(numStates * 0x300))]; } } private void InitFastModel() { const ushort probInit = (ushort)(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL >> 1); - Array.Fill(_fIsMatch, probInit); - Array.Fill(_fIsRep, probInit); - Array.Fill(_fIsRepG0, probInit); - Array.Fill(_fIsRepG1, probInit); - Array.Fill(_fIsRepG2, probInit); - Array.Fill(_fIsRep0Long, probInit); - Array.Fill(_fPosSlot, probInit); - Array.Fill(_fPosDecoders, probInit); - Array.Fill(_fPosAlign, probInit); - Array.Fill(_fLenProbs, probInit); - Array.Fill(_fRepLenProbs, probInit); - Array.Fill(_fLiteral, probInit); + Array.Fill( + _fLiteral, + probInit, + 0, + LiteralOffset + (0x300 << (_fLiteralNumPrevBits + _fLiteralNumPosBits)) + ); } // Mutable range-coder state threaded through the decode helpers below as a single `ref` @@ -269,21 +266,19 @@ public partial class Decoder fixed (byte* pDic = dic) fixed (byte* pIn = inBuf) - fixed ( - ushort* pIsMatch = _fIsMatch, - pIsRep = _fIsRep, - pIsRepG0 = _fIsRepG0, - pIsRepG1 = _fIsRepG1, - pIsRepG2 = _fIsRepG2, - pIsRep0Long = _fIsRep0Long, - pPosSlot = _fPosSlot, - pPosDecoders = _fPosDecoders, - pPosAlign = _fPosAlign, - pLenProbs = _fLenProbs, - pRepLenProbs = _fRepLenProbs, - pLiteral = _fLiteral - ) + fixed (ushort* pIsMatch = _fLiteral) { + var pIsRep = pIsMatch + IsRepOffset; + var pIsRepG0 = pIsMatch + IsRepG0Offset; + var pIsRepG1 = pIsMatch + IsRepG1Offset; + var pIsRepG2 = pIsMatch + IsRepG2Offset; + var pIsRep0Long = pIsMatch + IsRep0LongOffset; + var pPosSlot = pIsMatch + PosSlotOffset; + var pPosDecoders = pIsMatch + PosDecodersOffset; + var pPosAlign = pIsMatch + PosAlignOffset; + var pLenProbs = pIsMatch + LenProbsOffset; + var pRepLenProbs = pIsMatch + RepLenProbsOffset; + var pLiteral = pIsMatch + LiteralOffset; var rs = new FastRangeState { Range = rangeDecoder._range, diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index 8cb96a2f..4e1de7f8 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading.Tasks; @@ -15,8 +16,30 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable public void Dispose() { - _outWindow?.Dispose(); - _outWindow = null; + try + { + _outWindow?.Dispose(); + } + finally + { + _outWindow = null; + ReleaseModels(); + } + } + + private void ReleaseModels() + { + _literalDecoder.ReleaseModels(); + scalarModelsReady = false; +#if !LEGACY_DOTNET + if (_fLiteral is not null) + { + ArrayPool.Shared.Return(_fLiteral); + _fLiteral = null; + } + _fLiteralNumPrevBits = _fLiteralNumPosBits = -1; + fastModelsReady = false; +#endif } private partial class LenDecoder @@ -137,16 +160,31 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { return; } + var numStates = (uint)1 << (numPrevBits + numPosBits); + var required = checked((int)(numStates * 0x300)); + // Allocate the reference-bearing wrappers before returning any old rental. + var coders = + _coders is null || _coders.Length < numStates || _coders.Length / 2 > numStates + ? new Decoder2[numStates] + : _coders; + // Do not retain a large property bank after switching to small contexts. + if (_models is null || _models.Length < required || _models.Length / 2 > required) + { + var replacement = ArrayPool.Shared.Rent(required); + if (_models is not null) + { + ArrayPool.Shared.Return(_models); + } + _models = replacement; + } + for (uint i = 0; i < numStates; i++) + { + coders[i].Create(_models, checked((int)(i * 0x300))); + } + _coders = coders; _numPosBits = numPosBits; _posMask = ((uint)1 << numPosBits) - 1; _numPrevBits = numPrevBits; - var numStates = (uint)1 << (_numPrevBits + _numPosBits); - _models = new BitDecoder[checked((int)(numStates * 0x300))]; - _coders = new Decoder2[numStates]; - for (uint i = 0; i < numStates; i++) - { - _coders[i].Create(_models, checked((int)(i * 0x300))); - } } public void Init() @@ -158,6 +196,16 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable } } + public void ReleaseModels() + { + if (_models is not null) + { + ArrayPool.Shared.Return(_models); + } + _models = null; + _coders = null; + } + private uint GetState(uint pos, byte prevByte) => ((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits)); @@ -174,28 +222,17 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable private OutWindow _outWindow; - private readonly BitDecoder[] _isMatchDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; - private readonly BitDecoder[] _isRepDecoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRep0LongDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; - - private readonly BitTreeDecoder[] _posSlotDecoder = new BitTreeDecoder[ - Base.K_NUM_LEN_TO_POS_STATES - ]; - private readonly BitDecoder[] _posDecoders = new BitDecoder[ - Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX - ]; - - private BitTreeDecoder _posAlignDecoder = new(Base.K_NUM_ALIGN_BITS); - - private readonly LenDecoder _lenDecoder = new(); - private readonly LenDecoder _repLenDecoder = new(); + private BitDecoder[] _isMatchDecoders; + private BitDecoder[] _isRepDecoders; + private BitDecoder[] _isRepG0Decoders; + private BitDecoder[] _isRepG1Decoders; + private BitDecoder[] _isRepG2Decoders; + private BitDecoder[] _isRep0LongDecoders; + private BitTreeDecoder[] _posSlotDecoder; + private BitDecoder[] _posDecoders; + private BitTreeDecoder _posAlignDecoder; + private LenDecoder _lenDecoder; + private LenDecoder _repLenDecoder; private readonly LiteralDecoder _literalDecoder = new(); @@ -218,10 +255,6 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable public Decoder() { _dictionarySize = -1; - for (var i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++) - { - _posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS); - } } #if !LEGACY_DOTNET @@ -302,6 +335,27 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable { return; } + if (_isMatchDecoders is null) + { + // A sync-only decoder never needs this scalar object graph. + _isMatchDecoders = new BitDecoder[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX]; + _isRepDecoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES]; + _isRep0LongDecoders = new BitDecoder[ + Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX + ]; + _posSlotDecoder = new BitTreeDecoder[Base.K_NUM_LEN_TO_POS_STATES]; + for (var i = 0; i < _posSlotDecoder.Length; i++) + { + _posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS); + } + _posDecoders = new BitDecoder[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX]; + _posAlignDecoder = new BitTreeDecoder(Base.K_NUM_ALIGN_BITS); + _lenDecoder = new LenDecoder(); + _repLenDecoder = new LenDecoder(); + } _literalDecoder.Create(literalPosBits, literalContextBits); _lenDecoder.Create(_posStateMask + 1); _repLenDecoder.Create(_posStateMask + 1); @@ -332,15 +386,19 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable } var rangeDecoder = new RangeCoder.Decoder(); - rangeDecoder.Init(inStream); - - Code(_dictionarySize, _outWindow, rangeDecoder); - - _outWindow.ReleaseStream(); - rangeDecoder.ReleaseStream(); - - _outWindow.Dispose(); - _outWindow = null; + try + { + rangeDecoder.Init(inStream); + Code(_dictionarySize, _outWindow, rangeDecoder); + _outWindow.ReleaseStream(); + } + finally + { + rangeDecoder.ReleaseStream(); + var window = _outWindow; + _outWindow = null; + window.Dispose(); + } } #if !LEGACY_DOTNET diff --git a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs index 451acafe..4c04e45e 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs @@ -491,7 +491,7 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert public bool IsShortRep() => (_backPrev == 0); } - private readonly Optimal[] _optimum = new Optimal[K_NUM_OPTS]; + private Optimal[] _optimum; private BinTree _matchFinder; private readonly RangeCoder.Encoder _rangeEncoder = new(); @@ -605,6 +605,11 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert public void Dispose() { + if (_optimum is not null) + { + ArrayPool.Shared.Return(_optimum); + _optimum = null; + } _literalEncoder.Dispose(); _matchFinder?.Dispose(); _matchFinder = null; @@ -614,6 +619,9 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert private void Init() { + // This large workspace belongs to the encoder, not each input block. + _optimum ??= ArrayPool.Shared.Rent((int)K_NUM_OPTS); + Array.Clear(_optimum, 0, (int)K_NUM_OPTS); BaseInit(); _rangeEncoder.Init(); diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 1a0bf424..50906d03 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -465,8 +465,9 @@ public partial class LzmaStream cancellationToken.ThrowIfCancellationRequested(); if (_encoder != null) { + using var input = new SharpCompress.IO.ReadOnlyMemoryStream(buffer); _position = await _encoder - .CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken) + .CodeAsync(input, false, cancellationToken) .ConfigureAwait(false); } } @@ -523,7 +524,17 @@ public partial class LzmaStream try { #pragma warning disable VSTHRD103 // Encoder.Dispose only returns buffers; finalization is separate. - _encoder?.Dispose(); + try + { + _encoder?.Dispose(); + } + finally + { + if (_decoder is not null) + { + await _decoder.DisposeAsync().ConfigureAwait(false); + } + } #pragma warning restore VSTHRD103 } finally diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 0bae5c38..f068ae0d 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -267,7 +267,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { try { - _encoder?.Dispose(); + try + { + _encoder?.Dispose(); + } + finally + { + _decoder?.Dispose(); + } } finally { diff --git a/src/SharpCompress/IO/ReadOnlyMemoryStream.cs b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs new file mode 100644 index 00000000..bf5024d7 --- /dev/null +++ b/src/SharpCompress/IO/ReadOnlyMemoryStream.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SharpCompress.IO; + +// Borrows caller memory only until the awaited encoder operation completes. +internal sealed class ReadOnlyMemoryStream(ReadOnlyMemory memory) : Stream +{ + private int position; + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => memory.Length; + public override long Position + { + get => position; + set => throw new NotSupportedException(); + } + + private int ReadMemory(Span buffer) + { + var count = Math.Min(buffer.Length, memory.Length - position); + memory.Span.Slice(position, count).CopyTo(buffer); + position += count; + return count; + } + + public override int Read(byte[] buffer, int offset, int count) => + ReadMemory(buffer.AsSpan(offset, count)); + + public override int ReadByte() => position < memory.Length ? memory.Span[position++] : -1; + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Read(buffer, offset, count)); + } + +#if !LEGACY_DOTNET + public override int Read(Span buffer) => ReadMemory(buffer); + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + cancellationToken.ThrowIfCancellationRequested(); + return new ValueTask(ReadMemory(buffer.Span)); + } +#endif + + public override void Flush() { } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); +} diff --git a/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs b/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs new file mode 100644 index 00000000..6c7ffb0a --- /dev/null +++ b/tests/SharpCompress.Test/Streams/LzmaScratchTests.cs @@ -0,0 +1,384 @@ +using System; +using System.Buffers; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common.SevenZip; +using SharpCompress.Compressors.LZMA; +using SharpCompress.Compressors.LZMA.Utilities; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class LzmaScratchTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Raw_Decode_Failure_Releases_The_Dictionary(bool useAsync) + { + using var decoder = new SharpCompress.Compressors.LZMA.Decoder(); + decoder.SetDecoderProperties(new byte[] { 0x5D, 0, 0, 1, 0 }); + using var input = new FailingInput(); + if (useAsync) + { + await Assert.ThrowsAsync(async () => + await decoder.CodeAsync(input, Stream.Null, 32, 4096, null!, CancellationToken.None) + ); + } + else + { + Assert.Throws(() => decoder.Code(input, Stream.Null, 32, 4096, null!)); + } + Assert.Null( + decoder + .GetType() + .GetField("_outWindow", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder) + ); + } + + private sealed class FailingInput : MemoryStream + { + public override int ReadByte() => throw new IOException("Read failure"); + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) => throw new IOException("Read failure"); + +#if !LEGACY_DOTNET + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) => throw new IOException("Read failure"); +#endif + } + + [Theory] + [InlineData(false)] +#if !LEGACY_DOTNET + [InlineData(true)] +#endif + public void Smaller_Literal_Properties_Release_Oversized_Banks(bool fast) + { + using var decoder = new SharpCompress.Compressors.LZMA.Decoder(); + object target = decoder; + var methodName = "CreateFastModel"; + var fieldName = "_fLiteral"; + if (!fast) + { + target = decoder + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder)!; + methodName = "Create"; + fieldName = "_models"; + } + var method = target + .GetType() + .GetMethod( + methodName, + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance + )!; + var field = target + .GetType() + .GetField(fieldName, BindingFlags.NonPublic | BindingFlags.Instance)!; + method.Invoke(target, new object[] { 0, 8 }); + var large = (Array)field.GetValue(target)!; + method.Invoke(target, new object[] { 0, 0 }); + var small = (Array)field.GetValue(target)!; + Assert.True(small.Length < large.Length); + Assert.InRange(small.Length, 768, 4096); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(7)] + [InlineData(8)] + [InlineData(9)] + [InlineData(17)] + public void Digests_Keep_Partial_Vector_And_Little_Endian_Encoding(int count) + { + var crcs = Enumerable + .Range(0, count) + .Select(i => i % 3 == 0 ? (uint?)(0x12345678u + (uint)i) : null) + .ToArray(); + using var expected = new MemoryStream(); + expected.WriteOptionalBoolVector(crcs.Select(c => c.HasValue).ToArray()); + foreach (var crc in crcs) + { + if (crc is { } value) + { + var bytes = new byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + expected.Write(bytes, 0, bytes.Length); + } + } + using var actual = new MemoryStream(); + new SevenZipDigestsWriter(crcs).Write(actual); + Assert.Equal(expected.ToArray(), actual.ToArray()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Partial_Decode_Disposal_Releases_Literal_Models(bool useAsync) + { + using var packed = new MemoryStream(); + var encoder = new Lzma2EncoderStream(packed, 1 << 16, 32); + var properties = encoder.Properties; + using (encoder) + { + encoder.Write(new byte[8192], 0, 8192); + } + packed.Position = 0; + using var stream = LzmaStream.Create( + properties, + packed, + packed.Length, + 8192, + leaveOpen: true + ); + var bytes = new byte[1]; + if (useAsync) + { + await stream.ReadAsync(bytes, 0, 1, CancellationToken.None); + } + else + { + stream.Read(bytes, 0, 1); + } + var decoder = stream + .GetType() + .GetField("_decoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(stream)!; + if (useAsync) + { + await stream.DisposeAsync(); + } + else + { + stream.Dispose(); + } + stream.Dispose(); + var literal = decoder + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder)!; + Assert.Null( + literal + .GetType() + .GetField("_models", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(literal) + ); +#if !LEGACY_DOTNET + Assert.Null( + decoder + .GetType() + .GetField("_fLiteral", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(decoder) + ); +#endif + Assert.True(packed.CanRead); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Lzma2_Multiple_Blocks_RoundTrip(bool useAsync, bool random) + { + var expected = new byte[4 * 1024 * 1024 + 17]; + if (random) + { + new Random(987).NextBytes(expected); + } + else + { + for (var i = 0; i < expected.Length; i++) + { + expected[i] = (byte)(i % 17); + } + } + using var packed = new MemoryStream(); + byte[] properties; + var encoder = new Lzma2EncoderStream(packed, 1 << 16, 32); + properties = encoder.Properties; + if (useAsync) + { + await using (encoder) + { + await encoder.WriteAsync(expected, 0, expected.Length, CancellationToken.None); + } + } + else + { + using (encoder) + { + encoder.Write(expected, 0, expected.Length); + } + } + var trial = encoder + .GetType() + .GetField("_trialDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(encoder)!; + var literals = trial + .GetType() + .GetField("_literalDecoder", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(trial)!; + Assert.Null( + literals + .GetType() + .GetField("_models", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(literals) + ); +#if !LEGACY_DOTNET + Assert.Null( + trial + .GetType() + .GetField("_fLiteral", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(trial) + ); +#endif + packed.Position = 0; + using var decoder = LzmaStream.Create( + properties, + packed, + packed.Length, + expected.Length, + leaveOpen: true + ); + using var actual = new MemoryStream(); + if (useAsync) + { + await decoder.CopyToAsync(actual, 65536, CancellationToken.None); + } + else + { + decoder.CopyTo(actual); + } + Assert.Equal(expected, actual.ToArray()); + Assert.True(packed.CanRead); + } + +#if !LEGACY_DOTNET + [Fact] + public async Task Lzma_Memory_Write_Borrows_Only_The_Requested_Slice() + { + var bytes = new byte[8192]; + new Random(654).NextBytes(bytes); + var expected = Enumerable.Range(0, 4).SelectMany(_ => bytes.Skip(7).Take(4096)).ToArray(); + using var memory = new NonArrayMemory(bytes); + using var packed = new MemoryStream(); + var encoder = LzmaStream.Create(new LzmaEncoderProperties(true, 1 << 16), false, packed); + var properties = encoder.Properties; + await using (encoder) + { + for (var i = 0; i < 4; i++) + { + await encoder.WriteAsync(memory.Memory.Slice(7, 4096), CancellationToken.None); + } + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + await Assert.ThrowsAnyAsync(async () => + await encoder.WriteAsync(bytes.AsMemory(), cancelled.Token) + ); + // Caller memory may be reused as soon as the awaited write completes, + // even though encoder finalization still has pending output. + Array.Clear(bytes, 0, bytes.Length); + } + packed.Position = 0; + using var decoder = LzmaStream.Create( + properties, + packed, + packed.Length, + 16384, + leaveOpen: true + ); + using var actual = new MemoryStream(); + await decoder.CopyToAsync(actual, 65536, CancellationToken.None); + Assert.Equal(expected, actual.ToArray()); + } +#endif + + private sealed class NonArrayMemory(byte[] bytes) : MemoryManager + { + public override Span GetSpan() => bytes; + + public override MemoryHandle Pin(int elementIndex = 0) => throw new NotSupportedException(); + + public override void Unpin() { } + + protected override void Dispose(bool disposing) { } + } + + [Fact] + public void Encoder_Workspace_Is_Lazy_And_Returned_On_Dispose() + { + using var encoder = new SharpCompress.Compressors.LZMA.Encoder(); + var field = encoder + .GetType() + .GetField("_optimum", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(encoder)); + using var input = new MemoryStream(new byte[4096]); + encoder.Code(input, Stream.Null, input.Length, -1, null!); + Assert.NotNull(field.GetValue(encoder)); + encoder.Dispose(); + encoder.Dispose(); + Assert.Null(field.GetValue(encoder)); + } + + [Fact] + public void FilesInfo_Streams_Names_And_Partial_Metadata() + { + var entries = Enumerable + .Range(0, 17) + .Select(i => new SevenZipWriteEntry + { + Name = i + "-\ud83d\ude00-\ud800-" + new string('x', i % 2 == 0 ? 300 : 5), + IsEmpty = true, + IsDirectory = i % 3 == 0, + ModificationTime = + i % 2 == 0 ? new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc) : null, + Attributes = i % 2 == 0 ? 32u : null, + }) + .ToArray(); + using var stream = new MemoryStream(); + new SevenZipFilesInfoWriter { Entries = entries }.Write(stream); + var bytes = stream.ToArray(); + var reader = new DataReader(bytes, 0, bytes.Length); + Assert.Equal(17, reader.ReadNum()); + while (true) + { + var property = (BlockType)reader.ReadByte(); + if (property == BlockType.End) + { + break; + } + var payload = new byte[reader.ReadNum()]; + reader.ReadBytes(payload, 0, payload.Length); + if (property == BlockType.Name) + { + Assert.Equal(0, payload[0]); + Assert.Equal( + Encoding.Unicode.GetBytes(string.Concat(entries.Select(e => e.Name + "\0"))), + payload.Skip(1) + ); + } + else if (property == BlockType.MTime || property == BlockType.WinAttributes) + { + Assert.Equal(new byte[] { 0, 0xAA, 0xAA, 0x80, 0 }, payload.Take(5)); + Assert.Equal(5 + 9 * (property == BlockType.MTime ? 8 : 4), payload.Length); + } + } + Assert.Equal(bytes.Length, reader.Offset); + } +}