Improve LZMA decoder using 7zip logics for modern dotnet

- Add opt-in Parallel decoding
- Add a new ASM-like decoder
- Add Unit tests for Parallel decoding
This commit is contained in:
Julian Xhokaxhiu
2026-08-02 18:08:51 +02:00
parent 187055e673
commit 55db16ecb3
25 changed files with 2180 additions and 11 deletions

View File

@@ -46,7 +46,14 @@ public partial class SevenZipArchive
var file = _database._files[i];
entries[i] = new SevenZipArchiveEntry(
this,
new SevenZipFilePart(stream, _database, i, file, ReaderOptions.ArchiveEncoding),
new SevenZipFilePart(
stream,
_database,
i,
file,
ReaderOptions.ArchiveEncoding,
ReaderOptions.EnableParallelism
),
ReaderOptions
);
}

View File

@@ -54,7 +54,8 @@ public partial class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, Sev
_database,
i,
file,
ReaderOptions.ArchiveEncoding
ReaderOptions.ArchiveEncoding,
ReaderOptions.EnableParallelism
),
ReaderOptions
);
@@ -190,7 +191,8 @@ public partial class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, Sev
_currentFolderStream = _archive._database!.GetFolderStream(
_archive.Volumes.Single().Stream,
folder!,
_archive._database.PasswordProvider
_archive._database.PasswordProvider,
Options.EnableParallelism
);
}

View File

@@ -0,0 +1,14 @@
namespace SharpCompress.Common.Options;
/// <summary>
/// Options for controlling whether SharpCompress may use multi-threaded processing.
/// </summary>
public interface IParallelismOptions
{
/// <summary>
/// When true, opts in to any optional parallel (multi-threaded) processing a compressor may
/// offer. Compressors that do not implement parallel processing ignore this setting. Default
/// is false (single-threaded processing), so behavior is unchanged unless explicitly enabled.
/// </summary>
bool EnableParallelism { get; set; }
}

View File

@@ -3,7 +3,11 @@ using SharpCompress.Providers;
namespace SharpCompress.Common.Options;
public interface IReaderOptions : IStreamOptions, IEncodingOptions, IProgressOptions
public interface IReaderOptions
: IStreamOptions,
IEncodingOptions,
IProgressOptions,
IParallelismOptions
{
/// <summary>
/// Look for RarArchive (Check for self-extracting archives or cases where RarArchive isn't at the start of the file)

View File

@@ -7,7 +7,11 @@ namespace SharpCompress.Common.Options;
/// <summary>
/// Options for configuring writer behavior when creating archives.
/// </summary>
public interface IWriterOptions : IStreamOptions, IEncodingOptions, IProgressOptions
public interface IWriterOptions
: IStreamOptions,
IEncodingOptions,
IProgressOptions,
IParallelismOptions
{
/// <summary>
/// The compression type to use for the archive.

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.LZMA.Utilities;
using SharpCompress.IO;
namespace SharpCompress.Common.SevenZip;
@@ -144,8 +145,30 @@ internal partial class ArchiveDatabase
return size;
}
internal Stream GetFolderStream(Stream stream, CFolder folder, IPasswordProvider pw)
internal Stream GetFolderStream(
Stream stream,
CFolder folder,
IPasswordProvider pw,
bool enableParallelism = false
)
{
#if !LEGACY_DOTNET
// Opportunistically decode the whole folder concurrently when it's a single, unchained,
// unencrypted LZMA2 stream backed by a real (seekable, positional-I/O-capable) file -- the
// only shape where independent per-block decoding is possible and worth the temp-file
// round-trip. Opt-in only (via ReaderOptions.EnableParallelism): any other shape
// (multi-coder chains, BCJ/delta filters, encryption, a non-file stream, a single-core
// machine, or too few independent restart points to make parallelizing worthwhile) safely
// falls back to the existing sequential decode below.
if (
enableParallelism
&& TryGetParallelDecodedFolderStream(stream, folder, out var parallelStream)
)
{
return parallelStream!;
}
#endif
var packStreamIndex = folder._firstPackStreamId;
var folderStartPackPos = GetFolderStreamPos(folder, 0);
var count = folder._packStreams.Count;
@@ -164,6 +187,101 @@ internal partial class ArchiveDatabase
);
}
#if !LEGACY_DOTNET
/// <summary>
/// Attempts to decode <paramref name="folder"/> entirely up front using
/// <see cref="Lzma2ParallelDecoder"/>, writing the result to a temp file and returning a
/// stream over it. Returns false (and leaves <paramref name="decodedStream"/> null) whenever
/// the folder's shape, the host stream, or the current hardware make parallel decoding
/// inapplicable or not worthwhile -- callers must fall back to sequential decoding.
/// </summary>
private bool TryGetParallelDecodedFolderStream(
Stream stream,
CFolder folder,
out Stream? decodedStream
)
{
decodedStream = null;
// Matches 7-Zip's own behavior: with a single available core there is nothing to gain
// from splitting work across threads, so skip straight to the sequential path.
if (Environment.ProcessorCount <= 1)
{
return false;
}
// Only a lone, unchained LZMA2 coder can be split into independently-decodable blocks.
// Any bind pairs (BCJ/delta filters, etc.) or additional coders (e.g. AES encryption)
// mean the folder's bytes depend on more than just this one LZMA2 stream.
if (
folder._coders.Count != 1
|| folder._coders[0]._methodId != CMethodId.K_LZMA2
|| folder._packStreams.Count != 1
|| folder._bindPairs.Count != 0
)
{
return false;
}
var props = folder._coders[0]._props;
if (props is null || props.Length == 0)
{
return false;
}
// Parallel decoding needs positional (random-access) reads from the source file and
// positional writes to the temp output file across multiple threads; a real, seekable
// file handle is required for that. Streams the archive was opened from may be wrapped
// (buffering, source-stream indirection, etc.), so unwrap looking for the underlying file.
var inputFile = stream as FileStream ?? (stream as IStreamStack)?.GetStream<FileStream>();
if (inputFile is null)
{
return false;
}
var folderIndex = _folders.IndexOf(folder);
var packStart = GetFolderStreamPos(folder, 0);
var packSize = GetFolderFullPackSize(folderIndex);
var unpackSize = folder.GetUnpackSize();
var blocks = Lzma2ParallelDecoder.TryScanBlocks(stream, packStart, packSize, unpackSize);
if (blocks is null || blocks.Count <= 1)
{
return false;
}
var tempFile = new FileStream(
Path.GetTempFileName(),
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
4096,
FileOptions.DeleteOnClose
);
try
{
Lzma2ParallelDecoder.DecodeBlocksParallel(
inputFile.SafeFileHandle,
props,
packStart,
blocks,
tempFile.SafeFileHandle,
Math.Min(Environment.ProcessorCount, Lzma2ParallelDecoder.MaxThreads)
);
}
catch
{
tempFile.Dispose();
throw;
}
tempFile.Position = 0;
decodedStream = tempFile;
return true;
}
#endif
// Cache used to avoid re-decoding a solid folder from scratch for every file it contains.
// Without this, extracting N files from the same solid folder decodes O(N^2) bytes since each
// file's stream was previously created fresh from the folder start and skipped forward.
@@ -181,7 +299,8 @@ internal partial class ArchiveDatabase
CFolder folder,
IPasswordProvider pw,
long skipSize,
long entrySize
long entrySize,
bool enableParallelism = false
)
{
if (_cachedFolder == folder && _cachedFolderStream != null)
@@ -211,7 +330,7 @@ internal partial class ArchiveDatabase
_cachedFolder = null;
}
var newStream = GetFolderStream(stream, folder, pw);
var newStream = GetFolderStream(stream, folder, pw, enableParallelism);
if (skipSize > 0)
{
newStream.Skip(skipSize);

View File

@@ -11,18 +11,21 @@ internal class SevenZipFilePart : FilePart
private CompressionType? _type;
private readonly Stream _stream;
private readonly ArchiveDatabase _database;
private readonly bool _enableParallelism;
internal SevenZipFilePart(
Stream stream,
ArchiveDatabase database,
int index,
CFileItem fileEntry,
IArchiveEncoding archiveEncoding
IArchiveEncoding archiveEncoding,
bool enableParallelism = false
)
: base(archiveEncoding)
{
_stream = stream;
_database = database;
_enableParallelism = enableParallelism;
Index = index;
Header = fileEntry;
if (Header.HasStream)
@@ -58,7 +61,8 @@ internal class SevenZipFilePart : FilePart
Folder!,
_database.PasswordProvider,
skipSize,
Header.Size
Header.Size,
_enableParallelism
);
return new ReadOnlySubStream(folderStream, Header.Size, leaveOpen: true);
}

View File

@@ -24,6 +24,35 @@ internal partial class OutWindow : IDisposable
public long Total => _total;
#if !LEGACY_DOTNET
// Fast-path accessors used by the local-variable LZMA decode loop (see
// LzmaDecoder.Fast.cs). CodeFast snapshots pos/total/buffer into locals for the whole
// decode call instead of going through PutByte/GetByte/CopyBlock (and re-reading the
// fields of this object) for every single output byte, mirroring how the reference
// 7-Zip C decoder caches dicPos/dic as locals for the duration of one decode call.
internal byte[] FastBuffer => _buffer;
internal int FastPos
{
get => _pos;
set => _pos = value;
}
internal long FastTotal
{
get => _total;
set => _total = value;
}
internal int FastWindowSize => _windowSize;
internal long FastLimit => _limit;
internal void FastFlush() => Flush();
internal void SetPendingFast(int distance, int len)
{
_pendingDist = distance;
_pendingLen = len;
}
#endif
public void Create(int windowSize)
{
if (windowSize <= 0)

View File

@@ -0,0 +1,293 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
#if !LEGACY_DOTNET
using Microsoft.Win32.SafeHandles;
#endif
namespace SharpCompress.Compressors.LZMA;
/// <summary>
/// A contiguous run of one or more independent LZMA2 chunks that can be decoded without any
/// state (dictionary / range-coder) carried over from a preceding chunk.
/// </summary>
internal readonly record struct Lzma2Block(
long PackOffset,
long PackLen,
long UnpackOffset,
long UnpackLen
);
/// <summary>
/// Splits a single LZMA2-coded stream into independently-decodable blocks and decodes them
/// concurrently. The block-splitting strategy mirrors 7-Zip's own multi-threaded LZMA2 decoder
/// (see the reference 7-Zip C source, Lzma2DecMt.c / MtDec.c): independent restart points
/// (LZMA2 control byte 0x01, or 0xE0-0xFF) are candidate block boundaries, but consecutive
/// restart segments are merged together until the accumulated block reaches at least
/// <see cref="MinBlockSize"/> decoded bytes -- 7-Zip's own comment for this is "we decode small
/// blocks in one thread" -- to avoid parallelization overhead for tiny fragments. Merging also
/// stops once a block would exceed <see cref="MaxBlockSize"/>, matching 7-Zip's outBlockMax
/// default (Lzma2DecMtProps_Init: 1&lt;&lt;28 / 256MB).
/// </summary>
internal static class Lzma2ParallelDecoder
{
// 7-Zip Lzma2DecMt.c (block-parse loop): "we decode small blocks in one thread" --
// if (t->dec.decoder.dicPos >= (1 << 14)) break;
internal const int MinBlockSize = 1 << 14;
// 7-Zip Lzma2DecMt.c: Lzma2DecMtProps_Init -> p->outBlockMax = 1 << 28
internal const long MaxBlockSize = 1 << 28;
// 7-Zip MtDec.h: #define MTDEC_THREADS_MAX 32
internal const int MaxThreads = 32;
/// <summary>
/// Header-only walk of the LZMA2 chunk stream (no LZMA decoding) that locates every
/// independent restart point and merges adjacent segments into blocks using the same size
/// thresholds 7-Zip's own MT decoder uses. Returns null if the stream doesn't parse as
/// well-formed LZMA2 or the parsed total doesn't match <paramref name="unpackSize"/>; callers
/// should fall back to ordinary sequential decoding in that case.
/// </summary>
internal static List<Lzma2Block>? TryScanBlocks(
Stream packStream,
long packStart,
long packSize,
long unpackSize
)
{
try
{
packStream.Position = packStart;
var blocks = new List<Lzma2Block>();
long packPos = 0;
long unpackPos = 0;
var blockPackStart = 0L;
var blockUnpackStart = 0L;
var blockUnpackSize = 0L;
var blockHasContent = false;
Span<byte> sizeBuf = stackalloc byte[2];
Span<byte> header = stackalloc byte[4];
while (packPos < packSize)
{
var chunkPackStart = packPos;
var control = packStream.ReadByte();
if (control < 0)
{
return null; // truncated stream
}
packPos++;
if (control == 0)
{
break; // end-of-stream marker
}
long chunkUnpackSize;
long chunkPackPayload;
bool isRestart;
if (control < 0x80)
{
// 0x01 = uncompressed chunk + dictionary reset, 0x02 = uncompressed, no reset
if (control > 2)
{
return null; // not a recognized LZMA2 control byte
}
if (!ReadFully(packStream, sizeBuf))
{
return null;
}
packPos += 2;
chunkUnpackSize = ((sizeBuf[0] << 8) | sizeBuf[1]) + 1;
chunkPackPayload = chunkUnpackSize;
isRestart = control == 1;
}
else
{
// 0x80-0xFF = compressed chunk; bit layout: 1uuuuu, u = high bits of unpack size
if (!ReadFully(packStream, header))
{
return null;
}
packPos += 4;
chunkUnpackSize =
(long)(((control & 0x1F) << 16) | (header[0] << 8) | header[1]) + 1;
chunkPackPayload = ((header[2] << 8) | header[3]) + 1;
// 0xA0-0xBF state reset, 0xC0-0xDF state reset + new props, 0xE0-0xFF also
// resets the dictionary -- only a dictionary reset is an independent restart.
isRestart = control >= 0xE0;
// 0xC0-0xFF (new-props flag, bits 6-5 = 1x) carries one extra properties byte
// right after the 4 size-field bytes, before the compressed payload begins.
if (control >= 0xC0)
{
if (packStream.ReadByte() < 0)
{
return null;
}
packPos++;
}
}
if (
isRestart
&& blockHasContent
&& (
blockUnpackSize >= MinBlockSize
|| blockUnpackSize + chunkUnpackSize > MaxBlockSize
)
)
{
blocks.Add(
new Lzma2Block(
blockPackStart,
chunkPackStart - blockPackStart,
blockUnpackStart,
blockUnpackSize
)
);
blockPackStart = chunkPackStart;
blockUnpackStart = unpackPos;
blockUnpackSize = 0;
blockHasContent = false;
}
packPos += chunkPackPayload;
if (packPos > packSize)
{
return null; // chunk claims more pack bytes than the folder actually has
}
packStream.Seek(chunkPackPayload, SeekOrigin.Current);
unpackPos += chunkUnpackSize;
blockUnpackSize += chunkUnpackSize;
blockHasContent = true;
}
if (unpackPos != unpackSize || packPos != packSize)
{
return null; // scan didn't land on the expected totals -- don't trust it
}
if (blockHasContent)
{
blocks.Add(
new Lzma2Block(
blockPackStart,
packPos - blockPackStart,
blockUnpackStart,
blockUnpackSize
)
);
}
// Guards against a single unmerged segment large enough to overflow a byte[] buffer
// (only possible with an extreme, effectively-pathological run with zero restarts).
foreach (var block in blocks)
{
if (block.PackLen > int.MaxValue - 16 || block.UnpackLen > int.MaxValue - 16)
{
return null;
}
}
return blocks;
}
catch (Exception)
{
return null;
}
}
private static bool ReadFully(Stream stream, Span<byte> buffer)
{
var total = 0;
while (total < buffer.Length)
{
var read = stream.Read(buffer.Slice(total));
if (read <= 0)
{
return false;
}
total += read;
}
return true;
}
#if !LEGACY_DOTNET
/// <summary>
/// Decodes every block concurrently, reading packed bytes positionally from
/// <paramref name="inputHandle"/> and writing decoded bytes positionally to
/// <paramref name="outputHandle"/>. Both handles must support positional (random) access
/// since blocks complete out of order across threads.
/// </summary>
internal static void DecodeBlocksParallel(
SafeFileHandle inputHandle,
byte[] lzma2Props,
long packStart,
IReadOnlyList<Lzma2Block> blocks,
SafeFileHandle outputHandle,
int maxDegreeOfParallelism
)
{
Parallel.ForEach(
blocks,
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
block =>
{
var packBuffer = new byte[block.PackLen];
ReadFullyAt(inputHandle, packBuffer, packStart + block.PackOffset);
using var packStream = new MemoryStream(packBuffer, writable: false);
using var lzma = LzmaStream.Create(
lzma2Props,
packStream,
-1,
block.UnpackLen,
leaveOpen: true
);
var buffer = new byte[Math.Min(block.UnpackLen, 1 << 20)];
long total = 0;
while (total < block.UnpackLen)
{
var toRead = (int)Math.Min(buffer.Length, block.UnpackLen - total);
var read = lzma.Read(buffer, 0, toRead);
if (read <= 0)
{
throw new EndOfStreamException(
$"LZMA2 block at unpack offset {block.UnpackOffset:N0} ended early after {total:N0}/{block.UnpackLen:N0} bytes."
);
}
RandomAccess.Write(
outputHandle,
buffer.AsSpan(0, read),
block.UnpackOffset + total
);
total += read;
}
}
);
}
private static void ReadFullyAt(SafeFileHandle handle, byte[] buffer, long fileOffset)
{
var total = 0;
while (total < buffer.Length)
{
var read = RandomAccess.Read(handle, buffer.AsSpan(total), fileOffset + total);
if (read <= 0)
{
throw new EndOfStreamException(
"Unexpected end of pack stream while reading an LZMA2 block."
);
}
total += read;
}
}
#endif
}

View File

@@ -0,0 +1,794 @@
#nullable disable
#if !LEGACY_DOTNET
using System;
using System.Runtime.CompilerServices;
using SharpCompress.Compressors.LZMA.LZ;
namespace SharpCompress.Compressors.LZMA;
// Fast LZMA decode path used on .NET targets that support it (everything except
// net48/netstandard2.0/netstandard2.1, see LEGACY_DOTNET). This mirrors the design of the
// reference LZMA SDK / 7-Zip C decoder (LzmaDec.c): probabilities are stored in flat ushort
// arrays (instead of arrays of BitDecoder structs reached through nested decoder objects),
// range/code/dictionary-position/input-buffer-position are kept in local variables (pinned
// with `fixed` and accessed through raw pointers, exactly like the C reference's `probs`/
// `dic`/`buf` locals) for the duration of the decode loop, instead of being re-read from
// object fields - and re-bounds-checked - on every bit. Compressed input is consumed from a
// buffered reader (RangeCoder.Decoder's fast buffer) instead of one virtual Stream.ReadByte()
// call per byte.
//
// These tables are separate from (and not kept in sync with) the BitDecoder-based tables used
// by the async decode path (LzmaDecoder.Async.cs), which remains unchanged on every target.
// Both sets are kept up to date by SetDecoderProperties/Init, so either decode path can be used
// on a given Decoder instance, as long as sync and async decoding are not interleaved on the
// same instance.
public partial class Decoder
{
private const int KNumMoveBitsFast = 5;
// Bias used by the branchless probability-update formula in DecodeBitFast (see there for
// the derivation): (1 << KNumMoveBitsFast) - 1.
private const int KBitModelOffsetFast = (1 << KNumMoveBitsFast) - 1;
// Choice/Choice2 flags followed by Low (16 posStates * 8 symbols), Mid (16 posStates * 8
// symbols) and High (256 symbols) trees, laid out contiguously rather than reusing 7-Zip's
// overlapping ASM-oriented addressing, to keep the arithmetic straightforward and safe.
private const int LenChoiceIndex = 0;
private const int LenChoice2Index = 1;
private const int LenLowBase = 2;
private const int LenLowStride = 1 << Base.K_NUM_LOW_LEN_BITS; // 8
private const int LenLowSize = (int)Base.K_NUM_POS_STATES_MAX * LenLowStride; // 128
private const int LenMidBase = LenLowBase + LenLowSize;
private const int LenMidStride = 1 << Base.K_NUM_MID_LEN_BITS; // 8
private const int LenMidSize = (int)Base.K_NUM_POS_STATES_MAX * LenMidStride; // 128
private const int LenHighBase = LenMidBase + LenMidSize;
private const int LenHighSize = 1 << Base.K_NUM_HIGH_LEN_BITS; // 256
private const int LenProbsSize = LenHighBase + LenHighSize;
private ushort[] _fIsMatch;
private ushort[] _fIsRep;
private ushort[] _fIsRepG0;
private ushort[] _fIsRepG1;
private ushort[] _fIsRepG2;
private ushort[] _fIsRep0Long;
private ushort[] _fPosSlot;
private ushort[] _fPosDecoders;
private ushort[] _fPosAlign;
private ushort[] _fLenProbs;
private ushort[] _fRepLenProbs;
private ushort[] _fLiteral;
private int _fLiteralNumPrevBits = -1;
private int _fLiteralNumPosBits = -1;
private uint _fLiteralPosMask;
private void CreateFastModel(int lp, int lc)
{
_fIsMatch ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX];
_fIsRep ??= new ushort[Base.K_NUM_STATES];
_fIsRepG0 ??= new ushort[Base.K_NUM_STATES];
_fIsRepG1 ??= new ushort[Base.K_NUM_STATES];
_fIsRepG2 ??= new ushort[Base.K_NUM_STATES];
_fIsRep0Long ??= new ushort[Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX];
_fPosSlot ??= new ushort[Base.K_NUM_LEN_TO_POS_STATES << Base.K_NUM_POS_SLOT_BITS];
_fPosDecoders ??= new ushort[Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX];
_fPosAlign ??= new ushort[1 << Base.K_NUM_ALIGN_BITS];
_fLenProbs ??= new ushort[LenProbsSize];
_fRepLenProbs ??= new ushort[LenProbsSize];
if (_fLiteralNumPrevBits != lc || _fLiteralNumPosBits != lp)
{
_fLiteralNumPrevBits = lc;
_fLiteralNumPosBits = lp;
_fLiteralPosMask = ((uint)1 << lp) - 1;
var numStates = (uint)1 << (lc + lp);
_fLiteral = new ushort[checked((int)(numStates * 0x300))];
}
}
private void InitFastModel()
{
const ushort probInit = (ushort)(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL >> 1);
Array.Fill(_fIsMatch, probInit);
Array.Fill(_fIsRep, probInit);
Array.Fill(_fIsRepG0, probInit);
Array.Fill(_fIsRepG1, probInit);
Array.Fill(_fIsRepG2, probInit);
Array.Fill(_fIsRep0Long, probInit);
Array.Fill(_fPosSlot, probInit);
Array.Fill(_fPosDecoders, probInit);
Array.Fill(_fPosAlign, probInit);
Array.Fill(_fLenProbs, probInit);
Array.Fill(_fRepLenProbs, probInit);
Array.Fill(_fLiteral, probInit);
}
// Mutable range-coder state threaded through the decode helpers below as a single `ref`
// parameter (instead of one `ref` per field), pinned to the RangeCoder.Decoder's fast
// input buffer for the duration of one CodeFast call. Consumed byte count is batched into
// RangeCoder.Decoder._total only when the local buffer is refilled/at call exit, instead
// of touching that field on every single byte.
private unsafe struct FastRangeState
{
public uint Range;
public uint Code;
public byte* InBuf;
public int InPos;
public int InLen;
public long Consumed;
public RangeCoder.Decoder RangeDecoder;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte ReadByte()
{
if (InPos >= InLen)
{
RangeDecoder.FastBufferPos = InPos;
RangeDecoder.AddTotal(Consumed);
Consumed = 0;
RangeDecoder.RefillFast();
InPos = RangeDecoder.FastBufferPos;
InLen = RangeDecoder.FastBufferLen;
}
Consumed++;
return InBuf[InPos++];
}
}
// Mutable dictionary/output-window state, pinned to the OutWindow's circular buffer for
// the duration of one CodeFast call. Mirrors dicPos/dic being plain locals in the
// reference 7-Zip C decoder instead of being re-read from the OutWindow object.
private unsafe struct FastOutState
{
public byte* Dic;
public int Pos;
public long Total;
public int WindowSize;
public long Limit;
public OutWindow OutWindow;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly byte GetByte(int distance)
{
var p = Pos - distance - 1;
if (p < 0)
{
p += WindowSize;
}
return Dic[p];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void PutByte(byte value)
{
Dic[Pos++] = value;
Total++;
if (Pos >= WindowSize)
{
// Rare (once per dictionary-size worth of output): sync locals back, let
// OutWindow flush the pending bytes out to the underlying stream and wrap
// its position, then reload the (now wrapped) position.
OutWindow.FastPos = Pos;
OutWindow.FastTotal = Total;
OutWindow.FastFlush();
Pos = OutWindow.FastPos;
}
}
public void CopyBlock(int distance, int len)
{
var rem = len;
var p = (distance < Pos ? Pos : Pos + WindowSize) - distance - 1;
var targetSize =
(Pos < WindowSize && Total < Limit) ? (int)Math.Min(rem, Limit - Total) : 0;
var sizeUntilWindowEnd = Math.Min(WindowSize - Pos, WindowSize - p);
var sizeUntilOverlap = Math.Abs(p - Pos);
var fastSize = Math.Min(Math.Min(sizeUntilWindowEnd, sizeUntilOverlap), targetSize);
if (fastSize >= 2)
{
Buffer.MemoryCopy(Dic + p, Dic + Pos, fastSize, fastSize);
Pos += fastSize;
p += fastSize;
Total += fastSize;
if (Pos >= WindowSize)
{
OutWindow.FastPos = Pos;
OutWindow.FastTotal = Total;
OutWindow.FastFlush();
Pos = OutWindow.FastPos;
}
rem -= fastSize;
}
while (rem > 0 && Pos < WindowSize && Total < Limit)
{
if (p >= WindowSize)
{
p = 0;
}
PutByte(Dic[p++]);
rem--;
}
OutWindow.SetPendingFast(distance, rem);
}
}
internal unsafe bool CodeFast(
int dictionarySize,
OutWindow outWindow,
RangeCoder.Decoder rangeDecoder
)
{
var dictionarySizeCheck = Math.Max(dictionarySize, 1);
outWindow.CopyPending();
var dic = outWindow.FastBuffer;
var inBuf = rangeDecoder.FastBufferArray;
bool result;
fixed (byte* pDic = dic)
fixed (byte* pIn = inBuf)
fixed (
ushort* pIsMatch = _fIsMatch,
pIsRep = _fIsRep,
pIsRepG0 = _fIsRepG0,
pIsRepG1 = _fIsRepG1,
pIsRepG2 = _fIsRepG2,
pIsRep0Long = _fIsRep0Long,
pPosSlot = _fPosSlot,
pPosDecoders = _fPosDecoders,
pPosAlign = _fPosAlign,
pLenProbs = _fLenProbs,
pRepLenProbs = _fRepLenProbs,
pLiteral = _fLiteral
)
{
var rs = new FastRangeState
{
Range = rangeDecoder._range,
Code = rangeDecoder._code,
InBuf = pIn,
InPos = rangeDecoder.FastBufferPos,
InLen = rangeDecoder.FastBufferLen,
Consumed = 0,
RangeDecoder = rangeDecoder,
};
var os = new FastOutState
{
Dic = pDic,
Pos = outWindow.FastPos,
Total = outWindow.FastTotal,
WindowSize = outWindow.FastWindowSize,
Limit = outWindow.FastLimit,
OutWindow = outWindow,
};
result = false;
while (os.Pos < os.WindowSize && os.Total < os.Limit)
{
var posState = (uint)os.Total & _posStateMask;
var stateIndex = (int)_state._index;
// (stateIndex << K_NUM_POS_STATES_BITS_MAX) + posState is used both as the
// IsMatch index and (numerically identical) the IsRep0Long/"short rep" index.
var matchIndex = (int)((stateIndex << Base.K_NUM_POS_STATES_BITS_MAX) + posState);
// prevByte/matchByte only depend on os.Pos/_rep0, both already fixed from the
// previous iteration - independent of the upcoming (unavoidably branchy, since
// it changes the whole loop-body shape) IsMatch dispatch below. Issuing these
// dictionary reads before that branch resolves lets their load latency overlap
// with the IsMatch bit-decode arithmetic instead of stalling right after the
// branch, where they'd otherwise sit on the critical path. (Tried hoisting the
// literal-tree base-index/root-probability computation the same way too, but it
// regressed ~1s - that computation is heavy enough that doing it unconditionally
// on every iteration, including non-literal ones, outweighs the latency-hiding
// benefit; kept lazy, computed only once IsMatch resolves to 0.)
var isCharState = _state.IsCharState();
var prevByte = os.GetByte(0);
var matchByte = isCharState ? (byte)0 : os.GetByte((int)_rep0);
// Split compare/update (mirrors ASM's IF_BIT_x_NOUP + deferred UPDATE_0/
// UPDATE_1): the probability write only happens once we know which arm we're
// in, interleaved with that arm's own independent setup work below, instead of
// being on the decode's own critical path.
var probIsMatch = pIsMatch[matchIndex];
var isMatchSymbol = DecodeBitFastNoUpdate(ref rs, probIsMatch, out var isMatchMask);
if (isMatchSymbol == 0)
{
UpdateProbFast(pIsMatch + matchIndex, probIsMatch, isMatchMask);
var literalP =
pLiteral + (int)GetFastLiteralBaseIndex((uint)os.Total, prevByte);
var firstIndex = isCharState
? 1u
: (uint)((((matchByte >> 7) & 1) + 1) << 8) + 1;
var firstProb = (uint)literalP[firstIndex];
var b = isCharState
? LiteralDecodeNormalFast(ref rs, literalP, firstProb)
: LiteralDecodeWithMatchByteFast(
ref rs,
literalP,
matchByte,
firstIndex,
firstProb
);
os.PutByte(b);
_state.UpdateChar();
continue;
}
UpdateProbFast(pIsMatch + matchIndex, probIsMatch, isMatchMask);
uint len;
if (DecodeBitFast(ref rs, pIsRep, stateIndex) == 1)
{
if (DecodeBitFast(ref rs, pIsRepG0, stateIndex) == 0)
{
if (DecodeBitFast(ref rs, pIsRep0Long, matchIndex) == 0)
{
_state.UpdateShortRep();
os.PutByte(os.GetByte((int)_rep0));
continue;
}
}
else
{
uint distance;
if (DecodeBitFast(ref rs, pIsRepG1, stateIndex) == 0)
{
distance = _rep1;
}
else
{
if (DecodeBitFast(ref rs, pIsRepG2, stateIndex) == 0)
{
distance = _rep2;
}
else
{
distance = _rep3;
_rep3 = _rep2;
}
_rep2 = _rep1;
}
_rep1 = _rep0;
_rep0 = distance;
}
len = LenDecodeFast(ref rs, pRepLenProbs, posState) + Base.K_MATCH_MIN_LEN;
_state.UpdateRep();
}
else
{
_rep3 = _rep2;
_rep2 = _rep1;
_rep1 = _rep0;
len = Base.K_MATCH_MIN_LEN + LenDecodeFast(ref rs, pLenProbs, posState);
_state.UpdateMatch();
var posSlot = BitTreeDecodeFast(
ref rs,
pPosSlot,
(int)(Base.GetLenToPosState(len) << Base.K_NUM_POS_SLOT_BITS),
Base.K_NUM_POS_SLOT_BITS
);
if (posSlot >= Base.K_START_POS_MODEL_INDEX)
{
var numDirectBits = (int)((posSlot >> 1) - 1);
_rep0 = (2 | (posSlot & 1)) << numDirectBits;
if (posSlot < Base.K_END_POS_MODEL_INDEX)
{
_rep0 += BitTreeReverseDecodeFast(
ref rs,
pPosDecoders,
(int)(_rep0 - posSlot - 1),
numDirectBits
);
}
else
{
_rep0 +=
DecodeDirectBitsFast(ref rs, numDirectBits - Base.K_NUM_ALIGN_BITS)
<< Base.K_NUM_ALIGN_BITS;
_rep0 += BitTreeReverseDecodeFast(
ref rs,
pPosAlign,
0,
Base.K_NUM_ALIGN_BITS
);
}
}
else
{
_rep0 = posSlot;
}
}
if (_rep0 >= os.Total || _rep0 >= dictionarySizeCheck)
{
if (_rep0 == 0xFFFFFFFF)
{
result = true;
break;
}
rangeDecoder._range = rs.Range;
rangeDecoder._code = rs.Code;
rangeDecoder.FastBufferPos = rs.InPos;
rangeDecoder.AddTotal(rs.Consumed);
outWindow.FastPos = os.Pos;
outWindow.FastTotal = os.Total;
throw new DataErrorException();
}
os.CopyBlock((int)_rep0, (int)len);
}
rangeDecoder._range = rs.Range;
rangeDecoder._code = rs.Code;
rangeDecoder.FastBufferPos = rs.InPos;
rangeDecoder.AddTotal(rs.Consumed);
outWindow.FastPos = os.Pos;
outWindow.FastTotal = os.Total;
}
return result;
}
// Core of DecodeBitFast, factored out so the bit-tree/literal decoders below can pass in a
// probability that was already loaded (prefetched) by the *previous* tree level instead of
// loading it here, and can read back `mask` to select their own prefetched next-level
// probability without an extra branch. `probSlot` must point at the array slot the caller
// read `prob` from, for the write-back.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe uint DecodeBitFastCore(
ref FastRangeState rs,
ushort* probSlot,
uint prob,
out uint mask
)
{
var range = rs.Range;
var bound = (range >> RangeCoder.BitDecoder.K_NUM_BIT_MODEL_TOTAL_BITS) * prob;
// Branchless bit decode + probability update, mirroring 7-Zip's ASM decoder
// (Asm/x86/LzmaDecOpt.asm) rather than the reference C decoder's data-dependent
// `if`: both possible next-states are computed unconditionally and combined via an
// all-ones/all-zeros mask (the software equivalent of the ASM's cmovae/cmovb),
// instead of branching on a comparison whose outcome is close to 50/50 for
// well-compressed data and is therefore poorly predicted by the CPU.
var symbol = rs.Code < bound ? 0u : 1u;
mask = (uint)-(int)symbol; // 0 for symbol 0, 0xFFFFFFFF for symbol 1
rs.Range = (bound & ~mask) | ((range - bound) & mask);
rs.Code -= bound & mask;
// UPDATE_0 (symbol 0) is `prob + ((K_BIT_MODEL_TOTAL - prob) >> 5)`; UPDATE_1
// (symbol 1) is `prob - (prob >> 5)`. KBitModelOffsetFast (31) is the standard LZMA
// bias that makes a single *arithmetic* (signed) shift reproduce UPDATE_1's
// logical-shift result from a target of 0, so both formulas collapse into one
// branchless expression selected by the same mask used above.
int target = (int)(
(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL & ~mask) | (uint)(KBitModelOffsetFast & mask)
);
*probSlot = (ushort)((int)prob + ((target - (int)prob) >> KNumMoveBitsFast));
if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE)
{
rs.Range <<= 8;
rs.Code = (rs.Code << 8) | rs.ReadByte();
}
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe uint DecodeBitFast(ref FastRangeState rs, ushort* probs, int index) =>
DecodeBitFastCore(ref rs, probs + index, probs[index], out _);
// Split form of DecodeBitFastCore used only for the outer dispatch bit (IsMatch): mirrors
// the ASM's IF_BIT_x_NOUP / UPDATE_0 / UPDATE_1 split, where the probability-model update is
// deferred past the (unavoidable, since literal vs. match are wholly different code) branch
// so it can be interleaved with independent work in each arm instead of sitting on the
// decode's own critical path.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint DecodeBitFastNoUpdate(ref FastRangeState rs, uint prob, out uint mask)
{
var range = rs.Range;
var bound = (range >> RangeCoder.BitDecoder.K_NUM_BIT_MODEL_TOTAL_BITS) * prob;
var symbol = rs.Code < bound ? 0u : 1u;
mask = (uint)-(int)symbol;
rs.Range = (bound & ~mask) | ((range - bound) & mask);
rs.Code -= bound & mask;
if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE)
{
rs.Range <<= 8;
rs.Code = (rs.Code << 8) | rs.ReadByte();
}
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void UpdateProbFast(ushort* probSlot, uint prob, uint mask)
{
int target = (int)(
(RangeCoder.BitDecoder.K_BIT_MODEL_TOTAL & ~mask) | (uint)(KBitModelOffsetFast & mask)
);
*probSlot = (ushort)((int)prob + ((target - (int)prob) >> KNumMoveBitsFast));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe uint BitTreeDecodeFast(
ref FastRangeState rs,
ushort* probs,
int baseIndex,
int numBits
)
{
var p = probs + baseIndex;
uint m = 1;
// The very first probability must still be loaded fresh; from then on `prob` always
// arrives pre-loaded from the previous level's prefetch below.
var prob = (uint)p[1];
for (var i = 0; i < numBits; i++)
{
var child0 = m << 1;
// Prefetch BOTH possible next-level probabilities now: which child we'll actually
// need depends on the bit we're about to decode, but the child *indices* only
// depend on `m`, which is already known - so these loads can proceed in parallel
// with (instead of only after) the branchless decode below, hiding their latency.
var hasNext = i + 1 < numBits;
uint probChild0 = 0,
probChild1 = 0;
if (hasNext)
{
probChild0 = p[child0];
probChild1 = p[child0 + 1];
}
var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask);
m = child0 + bit;
if (hasNext)
{
prob = (probChild0 & ~mask) | (probChild1 & mask);
}
}
return m - ((uint)1 << numBits);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe uint BitTreeReverseDecodeFast(
ref FastRangeState rs,
ushort* probs,
int baseIndex,
int numBits
)
{
var p = probs + baseIndex;
uint m = 1;
var prob = (uint)p[1];
uint symbol = 0;
for (var i = 0; i < numBits; i++)
{
var child0 = m << 1;
var hasNext = i + 1 < numBits;
uint probChild0 = 0,
probChild1 = 0;
if (hasNext)
{
probChild0 = p[child0];
probChild1 = p[child0 + 1];
}
var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask);
symbol |= bit << i;
m = child0 + bit;
if (hasNext)
{
prob = (probChild0 & ~mask) | (probChild1 & mask);
}
}
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint DecodeDirectBitsFast(ref FastRangeState rs, int numTotalBits)
{
uint result = 0;
for (var i = numTotalBits; i > 0; i--)
{
rs.Range >>= 1;
var t = (rs.Code - rs.Range) >> 31;
rs.Code -= rs.Range & (t - 1);
result = (result << 1) | (1 - t);
if (rs.Range < RangeCoder.Decoder.K_TOP_VALUE)
{
rs.Code = (rs.Code << 8) | rs.ReadByte();
rs.Range <<= 8;
}
}
return result;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe uint LenDecodeFast(ref FastRangeState rs, ushort* probs, uint posState)
{
if (DecodeBitFast(ref rs, probs, LenChoiceIndex) == 0)
{
return BitTreeDecodeFast(
ref rs,
probs,
LenLowBase + (int)posState * LenLowStride,
Base.K_NUM_LOW_LEN_BITS
);
}
var symbol = Base.K_NUM_LOW_LEN_SYMBOLS;
if (DecodeBitFast(ref rs, probs, LenChoice2Index) == 0)
{
symbol += BitTreeDecodeFast(
ref rs,
probs,
LenMidBase + (int)posState * LenMidStride,
Base.K_NUM_MID_LEN_BITS
);
}
else
{
symbol += Base.K_NUM_MID_LEN_SYMBOLS;
symbol += BitTreeDecodeFast(ref rs, probs, LenHighBase, Base.K_NUM_HIGH_LEN_BITS);
}
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private uint GetFastLiteralBaseIndex(uint pos, byte prevByte) =>
(
((pos & _fLiteralPosMask) << _fLiteralNumPrevBits)
+ (uint)(prevByte >> (8 - _fLiteralNumPrevBits))
) * 0x300;
// baseP/firstProb (root of the literal tree, index 1) are precomputed by the caller -
// pos/prevByte are already known before the IsMatch decode resolves, so hoisting this load
// out lets it overlap with the IsMatch bit-decode instead of stalling right after it.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe byte LiteralDecodeNormalFast(
ref FastRangeState rs,
ushort* baseP,
uint firstProb
)
{
var p = baseP;
// Same one-level-ahead child-probability prefetch as BitTreeDecodeFast (see there),
// applied to the hottest call site in the decoder: one 8-bit literal-tree decode per
// literal byte. Manually unrolled (mirroring the ASM's explicit BIT_0/BIT_1x7
// sequence, as opposed to a loop) because RyuJIT does not inline methods containing
// a loop - leaving this as a `for` loop, even with AggressiveInlining, forces this
// call to remain a real call boundary, which makes `rs` address-exposed and prevents
// the JIT from keeping Range/Code in registers for the rest of the CodeFast loop.
uint m = 1;
var prob = firstProb;
var child0 = m << 1;
var probChild0 = p[child0];
var probChild1 = p[child0 + 1];
var bit = DecodeBitFastCore(ref rs, p + m, prob, out var mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
probChild0 = p[child0];
probChild1 = p[child0 + 1];
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
prob = (probChild0 & ~mask) | (probChild1 & mask);
child0 = m << 1;
bit = DecodeBitFastCore(ref rs, p + m, prob, out mask);
m = child0 + bit;
return (byte)m;
}
// Shared plain-literal-tree tail used once a matched-literal decode diverges from
// matchByte: from that point on there's no more matched-zone offset, so this is the same
// decode as LiteralDecodeNormalFast's inner step, just resuming mid-tree from `symbol`.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe byte LiteralDecodeTailFast(ref FastRangeState rs, ushort* p, uint symbol)
{
while (symbol < 0x100)
{
var bit = DecodeBitFastCore(ref rs, p + symbol, p[symbol], out _);
symbol = (symbol << 1) | bit;
}
return (byte)symbol;
}
// baseP/firstIndex/firstProb are precomputed by the caller for the same reason as in
// LiteralDecodeNormalFast above (pos/prevByte/matchByte are all already known before the
// IsMatch decode resolves).
private static unsafe byte LiteralDecodeWithMatchByteFast(
ref FastRangeState rs,
ushort* baseP,
byte matchByte,
uint firstIndex,
uint firstProb
)
{
var p = baseP;
// Same one-level-ahead prefetch as BitTreeDecodeFast, adapted for the matched-literal
// zone: matchByte's bits are all known upfront (it comes from the dictionary, not from
// decoding), so the *next* iteration's matched-zone index can be computed from it
// immediately, independent of this iteration's decoded bit.
uint symbol = 1;
var matchBit = (uint)(matchByte >> 7) & 1;
matchByte <<= 1;
var index = (int)firstIndex;
var prob = firstProb;
for (var i = 0; i < 8; i++)
{
var nextMatchBit = (uint)(matchByte >> 7) & 1;
matchByte <<= 1;
var hasNext = i < 7;
var child0 = (uint)((1 + nextMatchBit) << 8) + (symbol << 1);
uint probChild0 = 0,
probChild1 = 0;
if (hasNext)
{
probChild0 = p[child0];
probChild1 = p[child0 + 1];
}
var bit = DecodeBitFastCore(ref rs, p + index, prob, out var mask);
symbol = (symbol << 1) | bit;
if (matchBit != bit)
{
return LiteralDecodeTailFast(ref rs, p, symbol);
}
matchBit = nextMatchBit;
if (hasNext)
{
index = (int)(child0 + bit);
prob = (probChild0 & ~mask) | (probChild1 & mask);
}
}
return (byte)symbol;
}
}
#endif

View File

@@ -331,6 +331,12 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
_outWindow = null;
}
#if !LEGACY_DOTNET
// On modern .NET targets, the hot decode loop uses the unsafe flat-array fast path
// (see LzmaDecoder.Fast.cs) instead of the per-array, method-call based loop below.
internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) =>
CodeFast(dictionarySize, outWindow, rangeDecoder);
#else
internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder)
{
var dictionarySizeCheck = Math.Max(dictionarySize, 1);
@@ -456,6 +462,7 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
}
return false;
}
#endif
public void SetDecoderProperties(byte[] properties) =>
SetDecoderProperties(properties.AsSpan());
@@ -477,6 +484,10 @@ public partial class Decoder : ICoder, ISetDecoderProperties, IDisposable
SetLiteralProperties(lp, lc);
SetPosBitsProperties(pb);
Init();
#if !LEGACY_DOTNET
CreateFastModel(lp, lc);
InitFastModel();
#endif
if (properties.Length >= 5)
{
_dictionarySize = 0;

View File

@@ -131,6 +131,13 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
lzma._rangeDecoder.Init(inputStream);
#if !LEGACY_DOTNET
// Bound the fast buffered reader to the known compressed size (when available) so
// it never reads past this entry's data even on unbounded/shared streams. When the
// size is unknown (e.g. Zip data-descriptor entries), RangeCoder.Decoder falls back
// to conservative per-byte reads on streams that cannot report a safe Length.
lzma._rangeDecoder.SetFastLimit(lzma._rangeDecoderLimit);
#endif
}
else
{
@@ -474,6 +481,13 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
}
_rangeDecoder.Init(_inputStream);
#if !LEGACY_DOTNET
// LZMA2 chunks share one underlying stream with the raw chunk-header bytes read
// above/below, so the buffered fast-read path (see RangeCoder.Decoder's fast
// buffer) must never physically read past this chunk's compressed size, or it
// would desynchronize the stream position for the next chunk header.
_rangeDecoder.SetFastLimit(_rangeDecoderLimit);
#endif
}
else if (control > 0x02)
{

View File

@@ -1,7 +1,11 @@
#nullable disable
using System;
using System.IO;
using System.Runtime.CompilerServices;
#if !LEGACY_DOTNET
using System.Buffers;
#endif
namespace SharpCompress.Compressors.LZMA.RangeCoder;
@@ -89,6 +93,33 @@ internal partial class Decoder
public long _total;
private byte[] _singleByteBuffer;
#if !LEGACY_DOTNET
// Upper bound (in terms of _total) that the fast buffered reader is allowed to physically
// read up to. -1 means unbounded. This matters for formats like LZMA2 where multiple
// independent chunks share one underlying stream: chunk headers are read directly from that
// stream between decode sessions, so the fast path must not read past the end of the
// current chunk's compressed data, or it would desynchronize the stream position for the
// next chunk header read. Set via SetFastLimit once the caller knows the chunk/stream size.
private long _fastLimit = -1;
// Whether it is safe to bulk-read ahead of the decoder even without a known _fastLimit.
// This is only true for streams that are guaranteed to self-clamp Read() to their own
// logical end and never return bytes that belong to something else - e.g. 7Zip's
// per-folder BufferedSubStream, which limits every Read() to its own remaining pack size.
// It is false for everything else, notably a shared streaming Zip reader stream with an
// unknown compressed size (data-descriptor entries): that stream keeps handing out bytes
// past the logical end of this LZMA stream (the next entry's header, etc.) with no self
// clamping and no way to give unread bytes back, so bulk-buffering there would
// desynchronize the stream. Note a stream reporting a queryable Length is NOT a reliable
// signal here: some wrapper streams (e.g. SharpCompressStream's ring-buffer mode used for
// over-read recording on non-seekable Zip streams) expose a Length without actually
// bounding Read() to the current logical stream's end. In the unsafe case we fall back to
// reading exactly one byte at a time, matching the legacy per-byte ReadByte() behavior.
private bool _fastBufferSafeUnbounded;
public void SetFastLimit(long limit) => _fastLimit = limit;
#endif
public void Init(Stream stream)
{
_stream = stream;
@@ -100,11 +131,105 @@ internal partial class Decoder
_code = (_code << 8) | (byte)_stream.ReadByte();
}
_total = 5;
#if !LEGACY_DOTNET
_fastLimit = -1;
_fastBufferPos = 0;
_fastBufferLen = 0;
_fastEndOfStream = false;
_fastBufferSafeUnbounded = stream is SharpCompress.IO.BufferedSubStream;
#endif
}
public void ReleaseStream() =>
public void ReleaseStream()
{
#if !LEGACY_DOTNET
ReleaseFastBuffer();
#endif
// Stream.ReleaseStream();
_stream = null;
}
#if !LEGACY_DOTNET
// Buffered input used only by the unsafe fast LZMA decode path (see LzmaDecoder.Fast.cs).
// Avoids issuing a virtual Stream.ReadByte() call per consumed byte, which otherwise
// dominates decode time (millions of calls for a typical archive).
private const int FastBufferSize = 1 << 16;
private byte[] _fastBuffer;
private int _fastBufferPos;
private int _fastBufferLen;
private bool _fastEndOfStream;
// The fast decode loop (LzmaDecoder.Fast.cs) pins this array and keeps its own local
// copies of position/length/consumed-byte-count for the duration of a decode call, only
// syncing back through these accessors when the local buffer is exhausted (rare) or when
// the decode call ends. This avoids reloading these fields from this object on every bit.
internal byte[] FastBufferArray => _fastBuffer ??= ArrayPool<byte>.Shared.Rent(FastBufferSize);
internal int FastBufferPos
{
get => _fastBufferPos;
set => _fastBufferPos = value;
}
internal int FastBufferLen => _fastBufferLen;
internal void AddTotal(long consumed) => _total += consumed;
internal void RefillFast() => FillFastBuffer();
private void FillFastBuffer()
{
_fastBuffer ??= ArrayPool<byte>.Shared.Rent(FastBufferSize);
if (_fastEndOfStream)
{
// Match legacy (byte)_stream.ReadByte() behavior: once the stream is
// exhausted, keep yielding 0xFF instead of throwing or re-reading.
_fastBufferPos = 0;
_fastBufferLen = 1;
_fastBuffer[0] = 0xFF;
return;
}
var requestSize = _fastBuffer.Length;
if (_fastLimit >= 0)
{
var remaining = _fastLimit - _total;
requestSize = remaining <= 0 ? 1 : (int)Math.Min(requestSize, remaining);
}
else if (!_fastBufferSafeUnbounded)
{
// No known limit and the underlying stream cannot report a bounded Length (e.g. a
// shared, forward-only stream such as a streaming Zip reader). Reading ahead here
// could consume bytes that belong to whatever comes after this logical LZMA stream
// (the next Zip entry's header, etc.) with no way to give them back. Fall back to
// requesting exactly one byte at a time so we never consume more than the decoder
// actually needs, matching the legacy per-byte ReadByte() behavior.
requestSize = 1;
}
var read = _stream.Read(_fastBuffer, 0, requestSize);
if (read <= 0)
{
_fastEndOfStream = true;
_fastBufferPos = 0;
_fastBufferLen = 1;
_fastBuffer[0] = 0xFF;
return;
}
_fastBufferPos = 0;
_fastBufferLen = read;
}
private void ReleaseFastBuffer()
{
if (_fastBuffer is not null)
{
ArrayPool<byte>.Shared.Return(_fastBuffer);
_fastBuffer = null;
}
_fastBufferPos = 0;
_fastBufferLen = 0;
_fastEndOfStream = false;
}
#endif
public void Normalize()
{

View File

@@ -143,6 +143,13 @@ public sealed record ReaderOptions : IReaderOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel decode (e.g. 7-Zip's automatic
/// parallel LZMA2 solid-folder decode). Formats that do not implement parallel decoding
/// ignore this setting. Default is false (sequential decode).
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Creates a new ReaderOptions instance with default values.
/// </summary>

View File

@@ -86,6 +86,14 @@ public static class ReaderOptionsExtensions
int? rewindableBufferSize
) => options with { RewindableBufferSize = rewindableBufferSize };
/// <summary>
/// Creates a copy with the specified EnableParallelism value.
/// </summary>
public static ReaderOptions WithEnableParallelism(
this ReaderOptions options,
bool enableParallelism
) => options with { EnableParallelism = enableParallelism };
/// <summary>
/// Creates a copy with the specified compression provider registry.
/// </summary>

View File

@@ -81,6 +81,12 @@ public sealed record GZipWriterOptions : IWriterOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel encode. No writer currently implements
/// parallel encoding; reserved for future use. Default is false.
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Creates a new GZipWriterOptions instance with default values.
/// </summary>

View File

@@ -69,6 +69,12 @@ public sealed record SevenZipWriterOptions : IWriterOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel encode. No writer currently implements
/// parallel encoding; reserved for future use. Default is false.
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Whether to compress the archive header itself using LZMA.
/// Default is true, matching standard 7-Zip behavior.

View File

@@ -56,6 +56,12 @@ public sealed record TarWriterOptions : IWriterOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel encode. No writer currently implements
/// parallel encoding; reserved for future use. Default is false.
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Indicates if archive should be finalized (by 2 empty blocks) on close.
/// </summary>

View File

@@ -69,6 +69,12 @@ public sealed record WriterOptions : IWriterOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel encode. No writer currently implements
/// parallel encoding; reserved for future use. Default is false.
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Creates a new WriterOptions instance with the specified compression type.
/// Compression level is automatically set based on the compression type.

View File

@@ -73,6 +73,14 @@ public static class WriterOptionsExtensions
int compressionLevel
) => options with { CompressionLevel = compressionLevel };
/// <summary>
/// Creates a copy with the specified EnableParallelism value.
/// </summary>
public static WriterOptions WithEnableParallelism(
this WriterOptions options,
bool enableParallelism
) => options with { EnableParallelism = enableParallelism };
/// <summary>
/// Creates a copy with the specified archive encoding.
/// </summary>

View File

@@ -73,6 +73,12 @@ public sealed record ZipWriterOptions : IWriterOptions
public CompressionProviderRegistry Providers { get; set; } =
CompressionProviderRegistry.Default;
/// <summary>
/// When true, opts in to a format's optional parallel encode. No writer currently implements
/// parallel encoding; reserved for future use. Default is false.
/// </summary>
public bool EnableParallelism { get; set; }
/// <summary>
/// Optional comment for the archive.
/// </summary>