starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-11-15 17:02:03 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Pool LZMA workspaces and streamline 7z metadata writing

Browse Source 
Reuse LZMA2 trial decoders and pooled compressed output, pool and consolidate decoder probability banks, lazily rent encoder optimization scratch, and avoid copying direct async memory writes.

Use bounded stack scratch and stream 7z metadata without temporary property buffers. Add ownership, failure, context-shrink, Unicode, and block-boundary regressions.

Document allocation profiles, the small-file LZMA2 throughput tradeoff, and rejection of exact-size match-finder arrays. Validation: 3,016 .NET 10 tests passed; all library targets, net48 tests, and performance project compile; formatting checks pass.
This commit is contained in:
Adam Hathcock
2026-10-20 08:28:23 +01:00
parent 030569611e
commit f41cc208cc
15 changed files with 1014 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/SharpCompress/Common/SevenZip/CFolder.cs
View File

@@ -138,7 +138,8 @@ internal class CFolder
}
}
var mask = new uint[kMaskSize];
Span<uint> mask = stackalloc uint[kMaskSize];
mask.Clear();
{
var inStreamToCoder = new List<int>();

232
src/SharpCompress/Common/SevenZip/SevenZipFilesInfo.cs
View File

@@ -1,8 +1,8 @@
using System;
using System.Buffers;
using System.IO;
using System.Text;
using SharpCompress.Compressors.LZMA.Utilities;
using SharpCompress.IO;
namespace SharpCompress.Common.SevenZip;
@@ -63,18 +63,13 @@ internal sealed class SevenZipFilesInfoWriter
private void WriteEmptyStreamProperty(Stream stream, int emptyStreamCount)
{
var emptyStreams = new bool[Entries.Length];
var emptyFiles = new bool[emptyStreamCount];
var hasEmptyFile = false;
var emptyIndex = 0;
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].IsEmpty || Entries[i].IsDirectory)
{
emptyStreams[i] = true;
var isEmptyFile = !Entries[i].IsDirectory;
emptyFiles[emptyIndex++] = isEmptyFile;
if (isEmptyFile)
{
hasEmptyFile = true;
@@ -83,147 +78,194 @@ internal sealed class SevenZipFilesInfoWriter
}
// kEmptyStream
WriteFileProperty(stream, BlockType.EmptyStream, s => s.WriteBoolVector(emptyStreams));
WritePropertyHeader(stream, BlockType.EmptyStream, (Entries.LongLength + 7) / 8);
WriteFlags(stream, BlockType.EmptyStream);
// kEmptyFile (only if there are actual empty files, not just directories)
if (hasEmptyFile)
{
WriteFileProperty(stream, BlockType.EmptyFile, s => s.WriteBoolVector(emptyFiles));
WritePropertyHeader(stream, BlockType.EmptyFile, ((long)emptyStreamCount + 7) / 8);
WriteFlags(stream, BlockType.EmptyFile);
}
}
private void WriteNameProperty(Stream stream)
{
WriteFileProperty(
stream,
BlockType.Name,
s =>
long size = 1;
var maxBytes = 0;
foreach (var entry in Entries)
{
var count = Encoding.Unicode.GetByteCount(entry.Name);
size = checked(size + count + 2);
maxBytes = Math.Max(maxBytes, count);
}
WritePropertyHeader(stream, BlockType.Name, size);
stream.WriteByte(0);
byte[]? rented = null;
#if !LEGACY_DOTNET
Span<byte> small = stackalloc byte[512];
#endif
try
{
foreach (var entry in Entries)
{
// External = 0 (inline)
s.WriteByte(0);
for (var i = 0; i < Entries.Length; i++)
#if !LEGACY_DOTNET
var count = Encoding.Unicode.GetByteCount(entry.Name);
if (count <= small.Length)
{
var nameBytes = Encoding.Unicode.GetBytes(Entries[i].Name);
s.Write(nameBytes);
// null terminator (2 bytes for UTF-16)
s.WriteByte(0);
s.WriteByte(0);
Encoding.Unicode.GetBytes(entry.Name.AsSpan(), small);
stream.Write(small.Slice(0, count));
}
else
#endif
{
rented ??= ArrayPool<byte>.Shared.Rent(maxBytes);
var written = Encoding.Unicode.GetBytes(
entry.Name,
0,
entry.Name.Length,
rented,
0
);
stream.Write(rented, 0, written);
}
stream.WriteByte(0);
stream.WriteByte(0);
}
);
}
finally
{
if (rented is not null)
{
ArrayPool<byte>.Shared.Return(rented);
}
}
}
private void WriteMTimeProperty(Stream stream)
{
var hasTimes = false;
var definedCount = 0;
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].ModificationTime != null)
if (Entries[i].ModificationTime is not null)
{
hasTimes = true;
break;
definedCount++;
}
}
if (!hasTimes)
if (definedCount == 0)
{
return;
}
WriteFileProperty(
WritePropertyHeader(
stream,
BlockType.MTime,
s =>
{
var defined = new bool[Entries.Length];
for (var i = 0; i < Entries.Length; i++)
{
defined[i] = Entries[i].ModificationTime != null;
}
s.WriteOptionalBoolVector(defined);
// External = 0 (inline)
s.WriteByte(0);
var buf = new byte[8];
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].ModificationTime is { } mtime)
{
var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc();
System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(
buf,
fileTime
);
s.Write(buf, 0, 8);
}
}
}
2L
+ (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8)
+ 8L * definedCount
);
stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0));
if (definedCount != Entries.Length)
{
WriteFlags(stream, BlockType.MTime);
}
stream.WriteByte(0);
Span<byte> buf = stackalloc byte[8];
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].ModificationTime is { } mtime)
{
var fileTime = (ulong)mtime.ToUniversalTime().ToFileTimeUtc();
System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(buf, fileTime);
stream.Write(buf);
}
}
}
private void WriteAttributesProperty(Stream stream)
{
var hasAttrs = false;
var definedCount = 0;
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].Attributes != null)
if (Entries[i].Attributes is not null)
{
hasAttrs = true;
break;
definedCount++;
}
}
if (!hasAttrs)
if (definedCount == 0)
{
return;
}
WriteFileProperty(
WritePropertyHeader(
stream,
BlockType.WinAttributes,
s =>
{
var defined = new bool[Entries.Length];
for (var i = 0; i < Entries.Length; i++)
{
defined[i] = Entries[i].Attributes != null;
}
s.WriteOptionalBoolVector(defined);
// External = 0 (inline)
s.WriteByte(0);
var buf = new byte[4];
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].Attributes is { } attrs)
{
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs);
s.Write(buf, 0, 4);
}
}
}
2L
+ (definedCount == Entries.Length ? 0 : (Entries.LongLength + 7) / 8)
+ 4L * definedCount
);
stream.WriteByte((byte)(definedCount == Entries.Length ? 1 : 0));
if (definedCount != Entries.Length)
{
WriteFlags(stream, BlockType.WinAttributes);
}
stream.WriteByte(0);
Span<byte> buf = stackalloc byte[4];
for (var i = 0; i < Entries.Length; i++)
{
if (Entries[i].Attributes is { } attrs)
{
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, attrs);
stream.Write(buf);
}
}
}
/// <summary>
/// Writes a file property block: PropertyID + size + data.
/// Size is computed by writing to a temporary buffer first.
/// Writes a property prefix without staging its computable payload in another stream.
/// </summary>
private static void WriteFileProperty(
Stream stream,
BlockType propertyId,
Action<Stream> writeData
)
private static void WritePropertyHeader(Stream stream, BlockType propertyId, long size)
{
using var dataStream = new PooledMemoryStream();
writeData(dataStream);
stream.WriteByte((byte)propertyId);
stream.WriteEncodedUInt64((ulong)dataStream.Length);
dataStream.Position = 0;
dataStream.CopyTo(stream);
stream.WriteEncodedUInt64((ulong)size);
}
private void WriteFlags(Stream stream, BlockType property)
{
byte mask = 0x80;
byte bits = 0;
foreach (var entry in Entries)
{
var empty = entry.IsEmpty || entry.IsDirectory;
if (property == BlockType.EmptyFile && !empty)
{
continue;
}
var defined = property switch
{
BlockType.EmptyStream => empty,
BlockType.EmptyFile => !entry.IsDirectory,
BlockType.MTime => entry.ModificationTime is not null,
BlockType.WinAttributes => entry.Attributes is not null,
_ => throw new InvalidOperationException(),
};
if (defined)
{
bits |= mask;
}
mask >>= 1;
if (mask == 0)
{
stream.WriteByte(bits);
mask = 0x80;
bits = 0;
}
}
if (mask != 0x80)
{
stream.WriteByte(bits);
}
}
}

38
src/SharpCompress/Common/SevenZip/SevenZipHeaderStructures.cs
View File

@@ -13,21 +13,43 @@ internal sealed class SevenZipDigestsWriter(uint?[] crcs)
public void Write(Stream stream)
{
var defined = new bool[CRCs.Length];
var allDefined = true;
for (var i = 0; i < CRCs.Length; i++)
{
defined[i] = CRCs[i] != null;
allDefined &= CRCs[i] is not null;
}
stream.WriteByte((byte)(allDefined ? 1 : 0));
if (!allDefined)
{
byte mask = 0x80;
byte bits = 0;
foreach (var crc in CRCs)
{
if (crc is not null)
{
bits |= mask;
}
mask >>= 1;
if (mask == 0)
{
stream.WriteByte(bits);
mask = 0x80;
bits = 0;
}
}
if (mask != 0x80)
{
stream.WriteByte(bits);
}
}
stream.WriteOptionalBoolVector(defined);
var buf = new byte[4];
Span<byte> buf = stackalloc byte[4];
for (var i = 0; i < CRCs.Length; i++)
{
if (CRCs[i] is { } crcValue)
{
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(buf, crcValue);
stream.Write(buf, 0, 4);
stream.Write(buf);
}
}
}
@@ -184,14 +206,14 @@ internal sealed class SevenZipUnPackInfoWriter
stream.WriteByte(attributes);
// Codec ID bytes (big-endian, most significant byte first)
var codecId = new byte[codecIdLength];
Span<byte> codecId = stackalloc byte[8];
var id = coder._methodId._id;
for (var i = codecIdLength - 1; i >= 0; i--)
{
codecId[i] = (byte)(id & 0xFF);
id >>= 8;
}
stream.Write(codecId, 0, codecIdLength);
stream.Write(codecId.Slice(0, codecIdLength));
if (isComplex)
{

37
src/SharpCompress/Common/SevenZip/SevenZipSignatureHeader.cs
View File

@@ -29,17 +29,18 @@ internal static class SevenZipSignatureHeaderWriter
/// </summary>
public static void WritePlaceholder(Stream stream)
{
var header = new byte[HeaderSize];
Span<byte> header = stackalloc byte[HeaderSize];
header.Clear();
// magic signature
Array.Copy(Signature, 0, header, 0, Signature.Length);
Signature.AsSpan().CopyTo(header);
// version: major=0, minor=2 (standard 7z format)
header[6] = 0;
header[7] = 2;
// remaining 24 bytes are zero (placeholder for CRC and StartHeader)
stream.Write(header, 0, header.Length);
stream.Write(header);
}
/// <summary>
@@ -79,11 +80,12 @@ internal static class SevenZipSignatureHeaderWriter
uint nextHeaderCrc
)
{
var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc);
Span<byte> header = stackalloc byte[HeaderSize];
BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc);
// Write at position 0
stream.Position = 0;
stream.Write(header, 0, header.Length);
stream.Write(header);
}
/// <summary>
@@ -98,24 +100,26 @@ internal static class SevenZipSignatureHeaderWriter
CancellationToken cancellationToken = default
)
{
var header = BuildFinalHeader(nextHeaderOffset, nextHeaderSize, nextHeaderCrc);
var header = new byte[HeaderSize];
BuildFinalHeader(header, nextHeaderOffset, nextHeaderSize, nextHeaderCrc);
// Write at position 0
stream.Position = 0;
await stream.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false);
}
private static byte[] BuildFinalHeader(
private static void BuildFinalHeader(
Span<byte> header,
ulong nextHeaderOffset,
ulong nextHeaderSize,
uint nextHeaderCrc
)
{
// Build StartHeader (20 bytes): NextHeaderOffset(8) + NextHeaderSize(8) + NextHeaderCRC(4)
var startHeader = new byte[20];
BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(0, 8), nextHeaderOffset);
BinaryPrimitives.WriteUInt64LittleEndian(startHeader.AsSpan(8, 8), nextHeaderSize);
BinaryPrimitives.WriteUInt32LittleEndian(startHeader.AsSpan(16, 4), nextHeaderCrc);
var startHeader = header.Slice(12, 20);
BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(0, 8), nextHeaderOffset);
BinaryPrimitives.WriteUInt64LittleEndian(startHeader.Slice(8, 8), nextHeaderSize);
BinaryPrimitives.WriteUInt32LittleEndian(startHeader.Slice(16, 4), nextHeaderCrc);
// CRC32 of StartHeader
var startHeaderCrc = Crc32Stream.Compute(
@@ -125,21 +129,14 @@ internal static class SevenZipSignatureHeaderWriter
);
// Assemble full 32-byte header
var header = new byte[HeaderSize];
// magic signature
Array.Copy(Signature, 0, header, 0, Signature.Length);
Signature.AsSpan().CopyTo(header);
// version
header[6] = 0;
header[7] = 2;
// StartHeaderCRC
BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(8, 4), startHeaderCrc);
// StartHeader
Array.Copy(startHeader, 0, header, 12, startHeader.Length);
return header;
BinaryPrimitives.WriteUInt32LittleEndian(header.Slice(8, 4), startHeaderCrc);
}
}

4
src/SharpCompress/Common/SevenZip/SevenZipWriteExtensions.cs
View File

@@ -17,7 +17,7 @@ internal static class SevenZipWriteExtensions
/// </summary>
public static int WriteEncodedUInt64(this Stream stream, ulong value)
{
var data = new byte[9];
Span<byte> data = stackalloc byte[9];
data[0] = 0xFF;
byte mask = 0x80;
var length = 1;
@@ -36,7 +36,7 @@ internal static class SevenZipWriteExtensions
mask >>= 1;
}
stream.Write(data, 0, length);
stream.Write(data.Slice(0, length));
return length;
}

65
src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs
View File

@@ -10,7 +10,7 @@ namespace SharpCompress.Compressors.LZMA;
/// <summary>
/// Write-only stream that compresses data using the LZMA2 framing format.
/// Buffers input, compresses in chunks, and writes LZMA2-framed output to the underlying stream.
/// Each chunk is independently compressed with a fresh LZMA encoder.
/// Each chunk is independently compressed with reset LZMA state and reusable workspaces.
/// </summary>
internal sealed class Lzma2EncoderStream : Stream
{
@@ -30,6 +30,7 @@ internal sealed class Lzma2EncoderStream : Stream
private readonly ArrayPool<byte> _bufferPool;
private readonly byte[] _properties;
private readonly Encoder _encoder;
private Decoder? _trialDecoder;
private int _bufferPosition;
private bool _isFirstChunk = true;
private bool _isDisposed;
@@ -239,6 +240,7 @@ internal sealed class Lzma2EncoderStream : Stream
// Returning the input buffer must not depend on successful output or encoder cleanup.
try
{
_trialDecoder?.Dispose();
_encoder.Dispose();
}
finally
@@ -259,7 +261,7 @@ internal sealed class Lzma2EncoderStream : Stream
_bufferPosition = 0;
// Try compressing the data
(byte[] Buffer, int Length) compressed;
PooledMemoryStream compressed;
try
{
compressed = CompressBlock(_buffer, uncompressedSize);
@@ -271,13 +273,14 @@ internal sealed class Lzma2EncoderStream : Stream
return;
}
using var compressedScope = compressed;
// Check if compressed output fits in a single chunk and is actually smaller
if (
compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE
&& compressed.Length < uncompressedData.Length
)
{
WriteCompressedChunk(uncompressedData.Length, compressed.Buffer, compressed.Length);
WriteCompressedChunk(uncompressedData.Length, compressed, (int)compressed.Length);
}
else
{
@@ -296,7 +299,7 @@ internal sealed class Lzma2EncoderStream : Stream
var uncompressedData = _buffer.AsMemory(0, uncompressedSize);
_bufferPosition = 0;
(byte[] Buffer, int Length) compressed;
PooledMemoryStream compressed;
try
{
compressed = CompressBlock(_buffer, uncompressedSize);
@@ -308,6 +311,7 @@ internal sealed class Lzma2EncoderStream : Stream
return;
}
using var compressedScope = compressed;
if (
compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE
&& compressed.Length < uncompressedData.Length
@@ -315,8 +319,8 @@ internal sealed class Lzma2EncoderStream : Stream
{
await WriteCompressedChunkAsync(
uncompressedData.Length,
compressed.Buffer,
compressed.Length,
compressed,
(int)compressed.Length,
cancellationToken
)
.ConfigureAwait(false);
@@ -328,23 +332,31 @@ internal sealed class Lzma2EncoderStream : Stream
}
}
private (byte[] Buffer, int Length) CompressBlock(byte[] data, int length)
private PooledMemoryStream CompressBlock(byte[] data, int length)
{
using var inputMs = new MemoryStream(data, 0, length, writable: false);
using var outputMs = new PooledMemoryStream();
_encoder.Code(inputMs, outputMs, length, -1, null);
var fullCompressed = outputMs.ToArray();
// The LZMA range encoder flush writes trailing bytes the decoder doesn't consume.
// Trial-decode to find the exact byte count the decoder needs, so the LZMA2
// chunk header reports a compressed size that matches what the decoder reads.
var consumed = FindConsumedBytes(fullCompressed, fullCompressed.Length, length);
return (fullCompressed, consumed);
var outputMs = new PooledMemoryStream();
try
{
_encoder.Code(inputMs, outputMs, length, -1, null);
outputMs.Position = 0;
// Trial decoding excludes range-encoder flush bytes from the chunk size.
var consumed = FindConsumedBytes(outputMs, (int)outputMs.Length, length);
outputMs.SetLength(consumed);
outputMs.Position = 0;
return outputMs;
}
catch
{
outputMs.Dispose();
// A failed trial must not carry dictionary/output state into the next block.
_trialDecoder?.Dispose();
_trialDecoder = null;
throw;
}
}
private int FindConsumedBytes(byte[] compressedData, int compressedSize, int uncompressedSize)
private int FindConsumedBytes(Stream input, int compressedSize, int uncompressedSize)
{
// Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32]
Span<byte> props = stackalloc byte[5];
@@ -354,10 +366,9 @@ internal sealed class Lzma2EncoderStream : Stream
props[3] = (byte)(_dictionarySize >> 16);
props[4] = (byte)(_dictionarySize >> 24);
using var decoder = new Decoder();
var decoder = _trialDecoder ??= new Decoder();
decoder.SetDecoderProperties(props);
using var input = new MemoryStream(compressedData, 0, compressedSize, writable: false);
decoder.Code(input, Stream.Null, compressedSize, uncompressedSize, null);
return (int)input.Position;
@@ -369,14 +380,14 @@ internal sealed class Lzma2EncoderStream : Stream
/// </summary>
private void WriteCompressedChunk(
int uncompressedSize,
byte[] compressedData,
PooledMemoryStream compressedData,
int compressedSize
)
{
var uncompSizeMinus1 = uncompressedSize - 1;
var compSizeMinus1 = compressedSize - 1;
// Each chunk is compressed independently with a fresh LZMA encoder,
// Each chunk is compressed independently with reset LZMA encoder state,
// so we must use 0xE0 (full reset: dictionary + state + properties) every time.
// The decoder uses outWindow.Total for literal context and posState;
// 0xE0 triggers outWindow.Reset() which zeros Total, matching the encoder's
@@ -393,12 +404,12 @@ internal sealed class Lzma2EncoderStream : Stream
// 0xE0 (>= 0xC0) requires properties byte
_output.WriteByte(_lzmaPropertiesByte);
_output.Write(compressedData, 0, compressedSize);
compressedData.WriteTo(_output);
}
private async ValueTask WriteCompressedChunkAsync(
int uncompressedSize,
byte[] compressedData,
PooledMemoryStream compressedData,
int compressedSize,
CancellationToken cancellationToken = default
)
@@ -423,9 +434,7 @@ internal sealed class Lzma2EncoderStream : Stream
{
ArrayPool<byte>.Shared.Return(header);
}
await _output
.WriteAsync(compressedData, 0, compressedSize, cancellationToken)
.ConfigureAwait(false);
await compressedData.CopyToAsync(_output, 65536, cancellationToken).ConfigureAwait(false);
}
/// <summary>

35
src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs
View File

@@ -13,10 +13,17 @@ public partial class Decoder : IAsyncDisposable
{
public async ValueTask DisposeAsync()
{
if (_outWindow is not null)
try
{
if (_outWindow is not null)
{
await _outWindow.DisposeAsync().ConfigureAwait(false);
}
}
finally
{
await _outWindow.DisposeAsync().ConfigureAwait(false);
_outWindow = null;
ReleaseModels();
}
}
@@ -157,16 +164,20 @@ public partial class Decoder : IAsyncDisposable
}
var rangeDecoder = new RangeCoder.Decoder();
await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false);
await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken)
.ConfigureAwait(false);
await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false);
rangeDecoder.ReleaseStream();
await _outWindow.DisposeAsync().ConfigureAwait(false);
_outWindow = null;
try
{
await rangeDecoder.InitAsync(inStream, cancellationToken).ConfigureAwait(false);
await CodeAsync(_dictionarySize, _outWindow, rangeDecoder, cancellationToken)
.ConfigureAwait(false);
await _outWindow.ReleaseStreamAsync(cancellationToken).ConfigureAwait(false);
}
finally
{
rangeDecoder.ReleaseStream();
var window = _outWindow;
_outWindow = null;
await window.DisposeAsync().ConfigureAwait(false);
}
}
internal async ValueTask<bool> CodeAsync(

97
src/SharpCompress/Compressors/LZMA/LzmaDecoder.Fast.cs
View File

@@ -2,6 +2,7 @@
#if !LEGACY_DOTNET
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using SharpCompress.Compressors.LZMA.LZ;
@@ -46,17 +47,22 @@ public partial class Decoder
private const int LenHighSize = 1 << Base.K_NUM_HIGH_LEN_BITS; // 256
private const int LenProbsSize = LenHighBase + LenHighSize;
private ushort[] _fIsMatch;
private ushort[] _fIsRep;
private ushort[] _fIsRepG0;
private ushort[] _fIsRepG1;
private ushort[] _fIsRepG2;
private ushort[] _fIsRep0Long;
private ushort[] _fPosSlot;
private ushort[] _fPosDecoders;
private ushort[] _fPosAlign;
private ushort[] _fLenProbs;
private ushort[] _fRepLenProbs;
private const int IsRepOffset = (int)Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX;
private const int IsRepG0Offset = IsRepOffset + (int)Base.K_NUM_STATES;
private const int IsRepG1Offset = IsRepG0Offset + (int)Base.K_NUM_STATES;
private const int IsRepG2Offset = IsRepG1Offset + (int)Base.K_NUM_STATES;
private const int IsRep0LongOffset = IsRepG2Offset + (int)Base.K_NUM_STATES;
private const int PosSlotOffset = IsRep0LongOffset + IsRepOffset;
private const int PosDecodersOffset =
PosSlotOffset + ((int)Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS);
private const int PosAlignOffset =
PosDecodersOffset + (int)(Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX);
private const int LenProbsOffset = PosAlignOffset + (1 << Base.K_NUM_ALIGN_BITS);
private const int RepLenProbsOffset = LenProbsOffset + LenProbsSize;
private const int LiteralOffset = RepLenProbsOffset + LenProbsSize;
// Fixed probabilities precede literals in one rental. For default properties the
// whole bank fits a 16 KiB bucket, avoiding a separate rounded-up literal rental.
private ushort[] _fLiteral;
private int _fLiteralNumPrevBits = -1;
@@ -65,43 +71,34 @@ public partial class Decoder
private void CreateFastModel(int lp, int lc)
{
_fIsMatch ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX];
_fIsRep ??= new ushort[Base.K_NUM_STATES];
_fIsRepG0 ??= new ushort[Base.K_NUM_STATES];
_fIsRepG1 ??= new ushort[Base.K_NUM_STATES];
_fIsRepG2 ??= new ushort[Base.K_NUM_STATES];
_fIsRep0Long ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX];
_fPosSlot ??= new ushort[Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS];
_fPosDecoders ??= new ushort[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX];
_fPosAlign ??= new ushort[1 << Base.K_NUM_ALIGN_BITS];
_fLenProbs ??= new ushort[LenProbsSize];
_fRepLenProbs ??= new ushort[LenProbsSize];
if (_fLiteralNumPrevBits != lc || _fLiteralNumPosBits != lp)
{
var numStates = (uint)1 << (lc + lp);
var required = checked(LiteralOffset + (int)(numStates * 0x300));
if (_fLiteral is null || _fLiteral.Length < required || _fLiteral.Length / 2 > required)
{
var replacement = ArrayPool<ushort>.Shared.Rent(required);
if (_fLiteral is not null)
{
ArrayPool<ushort>.Shared.Return(_fLiteral);
}
_fLiteral = replacement;
}
_fLiteralNumPrevBits = lc;
_fLiteralNumPosBits = lp;
_fLiteralPosMask = ((uint)1 << lp) - 1;
var numStates = (uint)1 << (lc + lp);
_fLiteral = new ushort[checked((int)(numStates * 0x300))];
}
}
private void InitFastModel()
{
const ushort probInit = (ushort)(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL >> 1);
Array.Fill(_fIsMatch, probInit);
Array.Fill(_fIsRep, probInit);
Array.Fill(_fIsRepG0, probInit);
Array.Fill(_fIsRepG1, probInit);
Array.Fill(_fIsRepG2, probInit);
Array.Fill(_fIsRep0Long, probInit);
Array.Fill(_fPosSlot, probInit);
Array.Fill(_fPosDecoders, probInit);
Array.Fill(_fPosAlign, probInit);
Array.Fill(_fLenProbs, probInit);
Array.Fill(_fRepLenProbs, probInit);
Array.Fill(_fLiteral, probInit);
Array.Fill(
_fLiteral,
probInit,
0,
LiteralOffset + (0x300 << (_fLiteralNumPrevBits + _fLiteralNumPosBits))
);
}
// Mutable range-coder state threaded through the decode helpers below as a single `ref`
@@ -269,21 +266,19 @@ public partial class Decoder
fixed (byte* pDic = dic)
fixed (byte* pIn = inBuf)
fixed (
ushort* pIsMatch = _fIsMatch,
pIsRep = _fIsRep,
pIsRepG0 = _fIsRepG0,
pIsRepG1 = _fIsRepG1,
pIsRepG2 = _fIsRepG2,
pIsRep0Long = _fIsRep0Long,
pPosSlot = _fPosSlot,
pPosDecoders = _fPosDecoders,
pPosAlign = _fPosAlign,
pLenProbs = _fLenProbs,
pRepLenProbs = _fRepLenProbs,
pLiteral = _fLiteral
)
fixed (ushort* pIsMatch = _fLiteral)
{
var pIsRep = pIsMatch + IsRepOffset;
var pIsRepG0 = pIsMatch + IsRepG0Offset;
var pIsRepG1 = pIsMatch + IsRepG1Offset;
var pIsRepG2 = pIsMatch + IsRepG2Offset;
var pIsRep0Long = pIsMatch + IsRep0LongOffset;
var pPosSlot = pIsMatch + PosSlotOffset;
var pPosDecoders = pIsMatch + PosDecodersOffset;
var pPosAlign = pIsMatch + PosAlignOffset;
var pLenProbs = pIsMatch + LenProbsOffset;
var pRepLenProbs = pIsMatch + RepLenProbsOffset;
var pLiteral = pIsMatch + LiteralOffset;
var rs = new FastRangeState
{
Range = rangeDecoder._range,

146
src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs
View File

@@ -1,6 +1,7 @@
#nullable disable
using System;
using System.Buffers;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Threading.Tasks;
@@ -15,8 +16,30 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
public void Dispose()
{
_outWindow?.Dispose();
_outWindow = null;
try
{
_outWindow?.Dispose();
}
finally
{
_outWindow = null;
ReleaseModels();
}
}
private void ReleaseModels()
{
_literalDecoder.ReleaseModels();
scalarModelsReady = false;
#if !LEGACY_DOTNET
if (_fLiteral is not null)
{
ArrayPool<ushort>.Shared.Return(_fLiteral);
_fLiteral = null;
}
_fLiteralNumPrevBits = _fLiteralNumPosBits = -1;
fastModelsReady = false;
#endif
}
private partial class LenDecoder
@@ -137,16 +160,31 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
{
return;
}
var numStates = (uint)1 << (numPrevBits + numPosBits);
var required = checked((int)(numStates * 0x300));
// Allocate the reference-bearing wrappers before returning any old rental.
var coders =
_coders is null || _coders.Length < numStates || _coders.Length / 2 > numStates
? new Decoder2[numStates]
: _coders;
// Do not retain a large property bank after switching to small contexts.
if (_models is null || _models.Length < required || _models.Length / 2 > required)
{
var replacement = ArrayPool<BitDecoder>.Shared.Rent(required);
if (_models is not null)
{
ArrayPool<BitDecoder>.Shared.Return(_models);
}
_models = replacement;
}
for (uint i = 0; i < numStates; i++)
{
coders[i].Create(_models, checked((int)(i * 0x300)));
}
_coders = coders;
_numPosBits = numPosBits;
_posMask = ((uint)1 << numPosBits) - 1;
_numPrevBits = numPrevBits;
var numStates = (uint)1 << (_numPrevBits + _numPosBits);
_models = new BitDecoder[checked((int)(numStates * 0x300))];
_coders = new Decoder2[numStates];
for (uint i = 0; i < numStates; i++)
{
_coders[i].Create(_models, checked((int)(i * 0x300)));
}
}
public void Init()
@@ -158,6 +196,16 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
}
}
public void ReleaseModels()
{
if (_models is not null)
{
ArrayPool<BitDecoder>.Shared.Return(_models);
}
_models = null;
_coders = null;
}
private uint GetState(uint pos, byte prevByte) =>
((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits));
@@ -174,28 +222,17 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
private OutWindow _outWindow;
private readonly BitDecoder[] _isMatchDecoders = new BitDecoder[
Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX
];
private readonly BitDecoder[] _isRepDecoders = new BitDecoder[Base.K_NUM_STATES];
private readonly BitDecoder[] _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES];
private readonly BitDecoder[] _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES];
private readonly BitDecoder[] _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES];
private readonly BitDecoder[] _isRep0LongDecoders = new BitDecoder[
Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX
];
private readonly BitTreeDecoder[] _posSlotDecoder = new BitTreeDecoder[
Base.K_NUM_LEN_TO_POS_STATES
];
private readonly BitDecoder[] _posDecoders = new BitDecoder[
Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX
];
private BitTreeDecoder _posAlignDecoder = new(Base.K_NUM_ALIGN_BITS);
private readonly LenDecoder _lenDecoder = new();
private readonly LenDecoder _repLenDecoder = new();
private BitDecoder[] _isMatchDecoders;
private BitDecoder[] _isRepDecoders;
private BitDecoder[] _isRepG0Decoders;
private BitDecoder[] _isRepG1Decoders;
private BitDecoder[] _isRepG2Decoders;
private BitDecoder[] _isRep0LongDecoders;
private BitTreeDecoder[] _posSlotDecoder;
private BitDecoder[] _posDecoders;
private BitTreeDecoder _posAlignDecoder;
private LenDecoder _lenDecoder;
private LenDecoder _repLenDecoder;
private readonly LiteralDecoder _literalDecoder = new();
@@ -218,10 +255,6 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
public Decoder()
{
_dictionarySize = -1;
for (var i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++)
{
_posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS);
}
}
#if !LEGACY_DOTNET
@@ -302,6 +335,27 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
{
return;
}
if (_isMatchDecoders is null)
{
// A sync-only decoder never needs this scalar object graph.
_isMatchDecoders = new BitDecoder[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX];
_isRepDecoders = new BitDecoder[Base.K_NUM_STATES];
_isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES];
_isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES];
_isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES];
_isRep0LongDecoders = new BitDecoder[
Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX
];
_posSlotDecoder = new BitTreeDecoder[Base.K_NUM_LEN_TO_POS_STATES];
for (var i = 0; i < _posSlotDecoder.Length; i++)
{
_posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS);
}
_posDecoders = new BitDecoder[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX];
_posAlignDecoder = new BitTreeDecoder(Base.K_NUM_ALIGN_BITS);
_lenDecoder = new LenDecoder();
_repLenDecoder = new LenDecoder();
}
_literalDecoder.Create(literalPosBits, literalContextBits);
_lenDecoder.Create(_posStateMask + 1);
_repLenDecoder.Create(_posStateMask + 1);
@@ -332,15 +386,19 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
}
var rangeDecoder = new RangeCoder.Decoder();
rangeDecoder.Init(inStream);
Code(_dictionarySize, _outWindow, rangeDecoder);
_outWindow.ReleaseStream();
rangeDecoder.ReleaseStream();
_outWindow.Dispose();
_outWindow = null;
try
{
rangeDecoder.Init(inStream);
Code(_dictionarySize, _outWindow, rangeDecoder);
_outWindow.ReleaseStream();
}
finally
{
rangeDecoder.ReleaseStream();
var window = _outWindow;
_outWindow = null;
window.Dispose();
}
}
#if !LEGACY_DOTNET

10
src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs
View File

@@ -491,7 +491,7 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert
public bool IsShortRep() => (_backPrev == 0);
}
private readonly Optimal[] _optimum = new Optimal[K_NUM_OPTS];
private Optimal[] _optimum;
private BinTree _matchFinder;
private readonly RangeCoder.Encoder _rangeEncoder = new();
@@ -605,6 +605,11 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert
public void Dispose()
{
if (_optimum is not null)
{
ArrayPool<Optimal>.Shared.Return(_optimum);
_optimum = null;
}
_literalEncoder.Dispose();
_matchFinder?.Dispose();
_matchFinder = null;
@@ -614,6 +619,9 @@ internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderPropert
private void Init()
{
// This large workspace belongs to the encoder, not each input block.
_optimum ??= ArrayPool<Optimal>.Shared.Rent((int)K_NUM_OPTS);
Array.Clear(_optimum, 0, (int)K_NUM_OPTS);
BaseInit();
_rangeEncoder.Init();

15
src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs
View File

@@ -465,8 +465,9 @@ public partial class LzmaStream
cancellationToken.ThrowIfCancellationRequested();
if (_encoder != null)
{
using var input = new SharpCompress.IO.ReadOnlyMemoryStream(buffer);
_position = await _encoder
.CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken)
.CodeAsync(input, false, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -523,7 +524,17 @@ public partial class LzmaStream
try
{
#pragma warning disable VSTHRD103 // Encoder.Dispose only returns buffers; finalization is separate.
_encoder?.Dispose();
try
{
_encoder?.Dispose();
}
finally
{
if (_decoder is not null)
{
await _decoder.DisposeAsync().ConfigureAwait(false);
}
}
#pragma warning restore VSTHRD103
}
finally

9
src/SharpCompress/Compressors/LZMA/LzmaStream.cs
View File

@@ -267,7 +267,14 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
{
try
{
_encoder?.Dispose();
try
{
_encoder?.Dispose();
}
finally
{
_decoder?.Dispose();
}
}
finally
{

67
src/SharpCompress/IO/ReadOnlyMemoryStream.cs Normal file
View File

@@ -0,0 +1,67 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace SharpCompress.IO;
// Borrows caller memory only until the awaited encoder operation completes.
internal sealed class ReadOnlyMemoryStream(ReadOnlyMemory<byte> memory) : Stream
{
private int position;
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => memory.Length;
public override long Position
{
get => position;
set => throw new NotSupportedException();
}
private int ReadMemory(Span<byte> buffer)
{
var count = Math.Min(buffer.Length, memory.Length - position);
memory.Span.Slice(position, count).CopyTo(buffer);
position += count;
return count;
}
public override int Read(byte[] buffer, int offset, int count) =>
ReadMemory(buffer.AsSpan(offset, count));
public override int ReadByte() => position < memory.Length ? memory.Span[position++] : -1;
public override Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
return Task.FromResult(Read(buffer, offset, count));
}
#if !LEGACY_DOTNET
public override int Read(Span<byte> buffer) => ReadMemory(buffer);
public override ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
return new ValueTask<int>(ReadMemory(buffer.Span));
}
#endif
public override void Flush() { }
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
}