From 4c6f773d2ba2d14223aead3abf3b017bfda7f52c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 2 Jun 2026 13:15:16 +0100 Subject: [PATCH] more allocation reductions --- .../Compressors/PPMd/H/ModelPPM.cs | 230 ++++++++++-------- .../Compressors/PPMd/H/PPMContext.cs | 10 +- .../Compressors/PPMd/H/SubAllocator.cs | 21 +- .../Compressors/PPMd/PpmdStream.cs | 4 + .../Compressors/Rar/UnpackV1/Unpack.Async.cs | 57 ++++- .../Compressors/Rar/UnpackV1/Unpack.cs | 101 +++++++- .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 100 ++++---- src/SharpCompress/packages.lock.json | 12 +- tests/SharpCompress.Performance/Program.cs | 4 +- 9 files changed, 362 insertions(+), 177 deletions(-) diff --git a/src/SharpCompress/Compressors/PPMd/H/ModelPPM.cs b/src/SharpCompress/Compressors/PPMd/H/ModelPPM.cs index 6405423a..565f10ca 100644 --- a/src/SharpCompress/Compressors/PPMd/H/ModelPPM.cs +++ b/src/SharpCompress/Compressors/PPMd/H/ModelPPM.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using System.IO; using System.Text; using System.Threading; @@ -10,18 +11,23 @@ using Decoder = SharpCompress.Compressors.LZMA.RangeCoder.Decoder; namespace SharpCompress.Compressors.PPMd.H; -internal class ModelPpm +internal class ModelPpm : IDisposable { + private const int SEE2_CONTEXT_ROWS = 25; + private const int SEE2_CONTEXT_COLUMNS = 16; + private const int SEE2_CONTEXT_SIZE = SEE2_CONTEXT_ROWS * SEE2_CONTEXT_COLUMNS; + private const int BIN_SUMM_ROWS = 128; + private const int BIN_SUMM_COLUMNS = 64; + private const int BIN_SUMM_SIZE = BIN_SUMM_ROWS * BIN_SUMM_COLUMNS; + private void InitBlock() { - for (var i = 0; i < 25; i++) + for (var i = 0; i < SEE2_CONTEXT_SIZE; i++) { - _see2Cont[i] = new See2Context[16]; - } - for (var i2 = 0; i2 < 128; i2++) - { - _binSumm[i2] = new int[64]; + _see2Cont[i] = new See2Context(); } + + _binSumm = ArrayPool.Shared.Rent(BIN_SUMM_SIZE); } public SubAllocator SubAlloc { get; } = new(); @@ -68,8 +74,6 @@ internal class ModelPpm set => _hiBitsFlag = value & 0xff; } - public virtual int[][] BinSumm => _binSumm; - internal RangeCoder Coder { get; private set; } internal State FoundState { get; private set; } @@ -95,9 +99,9 @@ internal class ModelPpm public const int MAX_FREQ = 124; - private readonly See2Context[][] _see2Cont = new See2Context[25][]; + private readonly See2Context[] _see2Cont = new See2Context[SEE2_CONTEXT_SIZE]; - private See2Context _dummySee2Cont; + private readonly See2Context _dummySee2Cont = new(); private PpmContext _minContext; //medContext @@ -112,18 +116,12 @@ internal class ModelPpm private readonly int[] _charMask = new int[256]; - private readonly int[] _ns2Indx = new int[256]; - - private readonly int[] _ns2BsIndx = new int[256]; - - private readonly int[] _hb2Flag = new int[256]; - // byte EscCount, PrevSuccess, HiBitsFlag; private int _escCount, _prevSuccess, _hiBitsFlag; - private readonly int[][] _binSumm = new int[128][]; // binary SEE-contexts + private int[] _binSumm; // binary SEE-contexts private static readonly int[] INIT_BIN_ESC = { @@ -137,6 +135,61 @@ internal class ModelPpm 0x6051, }; + private static readonly int[] NS2_INDX = CreateNs2Indx(); + + private static readonly int[] NS2_BS_INDX = CreateNs2BsIndx(); + + private static readonly int[] HB2_FLAG = CreateHb2Flag(); + + private static int[] CreateNs2Indx() + { + var result = new int[256]; + int i, + k, + m, + step; + for (i = 0; i < 3; i++) + { + result[i] = i; + } + for (m = i, k = 1, step = 1; i < 256; i++) + { + result[i] = m; + if (--k == 0) + { + k = ++step; + m++; + } + } + return result; + } + + private static int[] CreateNs2BsIndx() + { + var result = new int[256]; + result[0] = 0; + result[1] = 2; + for (var j = 0; j < 9; j++) + { + result[2 + j] = 4; + } + for (var j = 0; j < 256 - 11; j++) + { + result[11 + j] = 6; + } + return result; + } + + private static int[] CreateHb2Flag() + { + var result = new int[256]; + for (var j = 0; j < 0x100 - 0x40; j++) + { + result[0x40 + j] = 0x08; + } + return result; + } + // Temp fields //UPGRADE_NOTE: Final was removed from the declaration of 'tempState1 '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'" private readonly State _tempState1 = new(null); @@ -171,6 +224,8 @@ internal class ModelPpm //UPGRADE_NOTE: Final was removed from the declaration of 'ps '. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1003'" private readonly int[] _ps = new int[MAX_O]; + private bool _isDisposed; + public ModelPpm() { InitBlock(); @@ -180,6 +235,23 @@ internal class ModelPpm //medContext = null; } + public void Dispose() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + SubAlloc.StopSubAllocator(); + var binSumm = _binSumm; + if (binSumm is not null) + { + _binSumm = null; + ArrayPool.Shared.Return(binSumm, clearArray: true); + } + } + private void RestartModelRare() { new Span(_charMask).Clear(); @@ -209,67 +281,30 @@ internal class ModelPpm state.SetSuccessor(0); } - for (var i = 0; i < 128; i++) + for (var i = 0; i < BIN_SUMM_ROWS; i++) { for (var k = 0; k < 8; k++) { - for (var m = 0; m < 64; m += 8) + for (var m = 0; m < BIN_SUMM_COLUMNS; m += 8) { - _binSumm[i][k + m] = BIN_SCALE - (INIT_BIN_ESC[k] / (i + 2)); + SetBinSumm(i, k + m, BIN_SCALE - (INIT_BIN_ESC[k] / (i + 2))); } } } - for (var i = 0; i < 25; i++) + for (var i = 0; i < SEE2_CONTEXT_ROWS; i++) { - for (var k = 0; k < 16; k++) + for (var k = 0; k < SEE2_CONTEXT_COLUMNS; k++) { - _see2Cont[i][k].Initialize((5 * i) + 10); + GetSee2Cont(i, k).Initialize((5 * i) + 10); } } } private void StartModelRare(int maxOrder) { - int i, - k, - m, - step; _escCount = 1; _maxOrder = maxOrder; RestartModelRare(); - - // Bug Fixed - _ns2BsIndx[0] = 0; - _ns2BsIndx[1] = 2; - for (var j = 0; j < 9; j++) - { - _ns2BsIndx[2 + j] = 4; - } - for (var j = 0; j < 256 - 11; j++) - { - _ns2BsIndx[11 + j] = 6; - } - for (i = 0; i < 3; i++) - { - _ns2Indx[i] = i; - } - for (m = i, k = 1, step = 1; i < 256; i++) - { - _ns2Indx[i] = m; - if ((--k) == 0) - { - k = ++step; - m++; - } - } - for (var j = 0; j < 0x40; j++) - { - _hb2Flag[j] = 0; - } - for (var j = 0; j < 0x100 - 0x40; j++) - { - _hb2Flag[0x40 + j] = 0x08; - } _dummySee2Cont.Shift = PERIOD_BITS; } @@ -320,14 +355,6 @@ internal class ModelPpm //medContext = new PPMContext(Heap); _maxContext = new PpmContext(Heap); FoundState = new State(Heap); - _dummySee2Cont = new See2Context(); - for (var i = 0; i < 25; i++) - { - for (var j = 0; j < 16; j++) - { - _see2Cont[i][j] = new See2Context(); - } - } StartModelRare(maxOrder); } return (_minContext.Address != 0); @@ -379,14 +406,6 @@ internal class ModelPpm _maxContext = new PpmContext(Heap); FoundState = new State(Heap); - _dummySee2Cont = new See2Context(); - for (var i = 0; i < 25; i++) - { - for (var j = 0; j < 16; j++) - { - _see2Cont[i][j] = new See2Context(); - } - } StartModelRare(maxOrder); } return _minContext.Address != 0; @@ -532,17 +551,24 @@ internal class ModelPpm return (symbol); } - public virtual See2Context[][] GetSee2Cont() => _see2Cont; + public virtual See2Context GetSee2Cont(int row, int column) => + _see2Cont[(row * SEE2_CONTEXT_COLUMNS) + column]; + + public virtual int GetBinSumm(int row, int column) => + _binSumm[(row * BIN_SUMM_COLUMNS) + column]; + + public virtual void SetBinSumm(int row, int column, int value) => + _binSumm[(row * BIN_SUMM_COLUMNS) + column] = value; public virtual void IncEscCount(int dEscCount) => EscCount += dEscCount; public virtual void IncRunLength(int dRunLength) => RunLength += dRunLength; - public virtual int[] GetHb2Flag() => _hb2Flag; + public virtual int[] GetHb2Flag() => HB2_FLAG; - public virtual int[] GetNs2BsIndx() => _ns2BsIndx; + public virtual int[] GetNs2BsIndx() => NS2_BS_INDX; - public virtual int[] GetNs2Indx() => _ns2Indx; + public virtual int[] GetNs2Indx() => NS2_INDX; private int CreateSuccessors(bool skip, State p1) { @@ -906,14 +932,6 @@ internal class ModelPpm //medContext = new PPMContext(Heap); _maxContext = new PpmContext(Heap); FoundState = new State(Heap); - _dummySee2Cont = new See2Context(); - for (var i = 0; i < 25; i++) - { - for (var j = 0; j < 16; j++) - { - _see2Cont[i][j] = new See2Context(); - } - } StartModelRare(maxOrder); return (_minContext.Address != 0); @@ -943,14 +961,6 @@ internal class ModelPpm //medContext = new PPMContext(Heap); _maxContext = new PpmContext(Heap); FoundState = new State(Heap); - _dummySee2Cont = new See2Context(); - for (var i = 0; i < 25; i++) - { - for (var j = 0; j < 16; j++) - { - _see2Cont[i][j] = new See2Context(); - } - } StartModelRare(maxOrder); return (_minContext.Address != 0); @@ -1010,7 +1020,7 @@ internal class ModelPpm { return -2; } - _hiBitsFlag = _hb2Flag[FoundState.Symbol]; + _hiBitsFlag = GetHb2Flag()[FoundState.Symbol]; decoder.Decode((uint)hiCnt, (uint)(_minContext.FreqData.SummFreq - hiCnt)); for (i = 0; i < 256; i++) { @@ -1031,12 +1041,15 @@ internal class ModelPpm _hiBitsFlag = GetHb2Flag()[FoundState.Symbol]; var off1 = rs.Freq - 1; var off2 = _minContext.GetArrayIndex(this, rs); - var bs = _binSumm[off1][off2]; + var bs = GetBinSumm(off1, off2); if (decoder.DecodeBit((uint)bs, 14) == 0) { byte symbol; - _binSumm[off1][off2] = - (bs + INTERVAL - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF; + SetBinSumm( + off1, + off2, + (bs + INTERVAL - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF + ); FoundState.Address = rs.Address; symbol = (byte)rs.Symbol; rs.IncrementFreq((rs.Freq < 128) ? 1 : 0); @@ -1046,7 +1059,7 @@ internal class ModelPpm return symbol; } bs = (bs - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF; - _binSumm[off1][off2] = bs; + SetBinSumm(off1, off2, bs); _initEsc = PpmContext.EXP_ESCAPE[Utility.URShift(bs, 10)]; int i; for (i = 0; i < 256; i++) @@ -1172,7 +1185,7 @@ internal class ModelPpm { return -2; } - _hiBitsFlag = _hb2Flag[FoundState.Symbol]; + _hiBitsFlag = GetHb2Flag()[FoundState.Symbol]; await decoder .DecodeAsync( (uint)hiCnt, @@ -1199,15 +1212,18 @@ internal class ModelPpm _hiBitsFlag = GetHb2Flag()[FoundState.Symbol]; var off1 = rs.Freq - 1; var off2 = _minContext.GetArrayIndex(this, rs); - var bs = _binSumm[off1][off2]; + var bs = GetBinSumm(off1, off2); if ( await decoder.DecodeBitAsync((uint)bs, 14, cancellationToken).ConfigureAwait(false) == 0 ) { byte symbol; - _binSumm[off1][off2] = - (bs + INTERVAL - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF; + SetBinSumm( + off1, + off2, + (bs + INTERVAL - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF + ); FoundState.Address = rs.Address; symbol = (byte)rs.Symbol; rs.IncrementFreq((rs.Freq < 128) ? 1 : 0); @@ -1217,7 +1233,7 @@ internal class ModelPpm return symbol; } bs = (bs - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF; - _binSumm[off1][off2] = bs; + SetBinSumm(off1, off2, bs); _initEsc = PpmContext.EXP_ESCAPE[Utility.URShift(bs, 10)]; int i; for (i = 0; i < 256; i++) diff --git a/src/SharpCompress/Compressors/PPMd/H/PPMContext.cs b/src/SharpCompress/Compressors/PPMd/H/PPMContext.cs index e4f98c71..a2a073d2 100644 --- a/src/SharpCompress/Compressors/PPMd/H/PPMContext.cs +++ b/src/SharpCompress/Compressors/PPMd/H/PPMContext.cs @@ -265,7 +265,7 @@ internal class PpmContext : Pointer model.HiBitsFlag = model.GetHb2Flag()[model.FoundState.Symbol]; var off1 = rs.Freq - 1; var off2 = GetArrayIndex(model, rs); - var bs = model.BinSumm[off1][off2]; + var bs = model.GetBinSumm(off1, off2); if (model.Coder.GetCurrentShiftCount(ModelPpm.TOT_BITS) < bs) { model.FoundState.Address = rs.Address; @@ -273,7 +273,7 @@ internal class PpmContext : Pointer model.Coder.SubRange.LowCount = 0; model.Coder.SubRange.HighCount = bs; bs = ((bs + ModelPpm.INTERVAL - GetMean(bs, ModelPpm.PERIOD_BITS, 2)) & 0xffff); - model.BinSumm[off1][off2] = bs; + model.SetBinSumm(off1, off2, bs); model.PrevSuccess = 1; model.IncRunLength(1); } @@ -281,7 +281,7 @@ internal class PpmContext : Pointer { model.Coder.SubRange.LowCount = bs; bs = (bs - GetMean(bs, ModelPpm.PERIOD_BITS, 2)) & 0xFFFF; - model.BinSumm[off1][off2] = bs; + model.SetBinSumm(off1, off2, bs); model.Coder.SubRange.HighCount = ModelPpm.BIN_SCALE; model.InitEsc = EXP_ESCAPE[Utility.URShift(bs, 10)]; model.NumMasked = 1; @@ -431,7 +431,7 @@ internal class PpmContext : Pointer idx2 += 2 * ((_freqData.SummFreq < 11 * numStats) ? 1 : 0); idx2 += 4 * ((model.NumMasked > diff) ? 1 : 0); idx2 += model.HiBitsFlag; - psee2C = model.GetSee2Cont()[idx1][idx2]; + psee2C = model.GetSee2Cont(idx1, idx2); model.Coder.SubRange.Scale = psee2C.Mean; } else @@ -457,7 +457,7 @@ internal class PpmContext : Pointer idx2 += 2 * ((_freqData.SummFreq < 11 * numStats) ? 1 : 0); idx2 += 4 * ((numMasked > nonMasked) ? 1 : 0); idx2 += model.HiBitsFlag; - psee2C = model.GetSee2Cont()[idx1][idx2]; + psee2C = model.GetSee2Cont(idx1, idx2); escFreq = psee2C.Mean; } else diff --git a/src/SharpCompress/Compressors/PPMd/H/SubAllocator.cs b/src/SharpCompress/Compressors/PPMd/H/SubAllocator.cs index e507f3c3..df6dfa60 100644 --- a/src/SharpCompress/Compressors/PPMd/H/SubAllocator.cs +++ b/src/SharpCompress/Compressors/PPMd/H/SubAllocator.cs @@ -1,11 +1,12 @@ #nullable disable using System; +using System.Buffers; using System.Text; namespace SharpCompress.Compressors.PPMd.H; -internal class SubAllocator +internal class SubAllocator : IDisposable { public virtual int FakeUnitsStart { @@ -124,21 +125,32 @@ internal class SubAllocator { if (_subAllocatorSize != 0) { + var heap = _heap; _subAllocatorSize = 0; - - //ArrayFactory.BYTES_FACTORY.recycle(heap); _heap = null; _heapStart = 1; // rarfree(HeapStart); + for (var i = 0; i < _freeList.Length; i++) + { + _freeList[i] = null; + } + // Free temp fields _tempRarNode = null; _tempRarMemBlock1 = null; _tempRarMemBlock2 = null; _tempRarMemBlock3 = null; + + if (heap is not null) + { + ArrayPool.Shared.Return(heap, clearArray: true); + } } } + public void Dispose() => StopSubAllocator(); + public virtual int GetAllocatedMemory() => _subAllocatorSize; public virtual bool StartSubAllocator(int saSize) @@ -159,7 +171,8 @@ internal class SubAllocator _tempMemBlockPos = realAllocSize; realAllocSize += RarMemBlock.SIZE; - _heap = new byte[realAllocSize]; + _heap = ArrayPool.Shared.Rent(realAllocSize); + new Span(_heap, 0, realAllocSize).Clear(); _heapStart = 1; _heapEnd = _heapStart + allocSize - UNIT_SIZE; _subAllocatorSize = t; diff --git a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs index 2ccf974f..b48cf4cf 100644 --- a/src/SharpCompress/Compressors/PPMd/PpmdStream.cs +++ b/src/SharpCompress/Compressors/PPMd/PpmdStream.cs @@ -182,6 +182,8 @@ public class PpmdStream : Stream, IAsyncDisposable { _model.EncodeBlock(_stream, Stream.Null, true); } + _modelH?.Dispose(); + _modelH = null; } base.Dispose(disposing); } @@ -202,6 +204,8 @@ public class PpmdStream : Stream, IAsyncDisposable { await _model.EncodeBlockAsync(_stream, new MemoryStream(), true).ConfigureAwait(false); } + _modelH?.Dispose(); + _modelH = null; #if !LEGACY_DOTNET || NETSTANDARD2_1 await base.DisposeAsync().ConfigureAwait(false); diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs index fd986955..adc53734 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs @@ -27,8 +27,16 @@ internal sealed partial class Unpack this.writeStream = writeStream; if (!fileHeader.IsSolid) { - Init(); + if (fileHeader.IsStored) + { + ReleaseWindow(); + } + else + { + Init(); + } } + suspended = false; await DoUnpackAsync(cancellationToken).ConfigureAwait(false); } @@ -40,6 +48,7 @@ internal sealed partial class Unpack await UnstoreFileAsync(cancellationToken).ConfigureAwait(false); return; } + switch (fileHeader.CompressionAlgorithm) { case 15: @@ -77,6 +86,7 @@ internal sealed partial class Unpack { break; } + code = code < destUnpSize ? code : (int)destUnpSize; await writeStream .WriteAsync(buffer, 0, code, cancellationToken) @@ -122,6 +132,7 @@ internal sealed partial class Unpack { return; } + if ( (!solid || !tablesRead) && !await ReadTablesAsync(cancellationToken).ConfigureAwait(false) @@ -155,12 +166,14 @@ internal sealed partial class Unpack { return; } + if (suspended) { FileExtracted = false; return; } } + if (unpBlockType == BlockTypes.BLOCK_PPM) { var Ch = await ppm.DecodeCharAsync(cancellationToken).ConfigureAwait(false); @@ -169,6 +182,7 @@ internal sealed partial class Unpack ppmError = true; break; } + if (Ch == PpmEscChar) { var NextCh = await ppm.DecodeCharAsync(cancellationToken).ConfigureAwait(false); @@ -178,20 +192,25 @@ internal sealed partial class Unpack { break; } + continue; } + if (NextCh == 2 || NextCh == -1) { break; } + if (NextCh == 3) { if (!await ReadVMCodePPMAsync(cancellationToken).ConfigureAwait(false)) { break; } + continue; } + if (NextCh == 4) { int Distance = 0, @@ -217,13 +236,16 @@ internal sealed partial class Unpack } } } + if (failed) { break; } + CopyString(Length + 32, Distance + 2); continue; } + if (NextCh == 5) { var Length = await ppm.DecodeCharAsync(cancellationToken) @@ -232,10 +254,12 @@ internal sealed partial class Unpack { break; } + CopyString(Length + 4, 1); continue; } } + window[unpPtr++] = (byte)Ch; continue; } @@ -246,6 +270,7 @@ internal sealed partial class Unpack window[unpPtr++] = (byte)Number; continue; } + if (Number >= 271) { var Length = LDecode[Number -= 271] + 3; @@ -266,6 +291,7 @@ internal sealed partial class Unpack Distance += ((Utility.URShift(GetBits(), (20 - Bits))) << 4); AddBits(Bits - 4); } + if (lowDistRepCount > 0) { lowDistRepCount--; @@ -307,30 +333,37 @@ internal sealed partial class Unpack CopyString(Length, Distance); continue; } + if (Number == 256) { if (!await ReadEndOfBlockAsync(cancellationToken).ConfigureAwait(false)) { break; } + continue; } + if (Number == 257) { if (!await ReadVMCodeAsync(cancellationToken).ConfigureAwait(false)) { break; } + continue; } + if (Number == 258) { if (lastLength != 0) { CopyString(lastLength, lastDist); } + continue; } + if (Number < 263) { var DistNum = Number - 259; @@ -339,6 +372,7 @@ internal sealed partial class Unpack { oldDist[I] = oldDist[I - 1]; } + oldDist[0] = Distance; var LengthNumber = this.decodeNumber(RD); @@ -348,10 +382,12 @@ internal sealed partial class Unpack Length += Utility.URShift(GetBits(), (16 - Bits)); AddBits(Bits); } + InsertLastMatch(Length, Distance); CopyString(Length, Distance); continue; } + if (Number < 272) { var Distance = SDDecode[Number -= 263] + 1; @@ -360,11 +396,13 @@ internal sealed partial class Unpack Distance += Utility.URShift(GetBits(), (16 - Bits)); AddBits(Bits); } + InsertOldDist(Distance); InsertLastMatch(2, Distance); CopyString(2, Distance); } } + await UnpWriteBufAsync(cancellationToken).ConfigureAwait(false); } @@ -379,11 +417,13 @@ internal sealed partial class Unpack { continue; } + if (flt.NextWindow) { flt.NextWindow = false; continue; } + var BlockStart = flt.BlockStart; var BlockLength = flt.BlockLength; if (((BlockStart - WrittenBorder) & PackDef.MAXWINMASK) < WriteSize) @@ -395,6 +435,7 @@ internal sealed partial class Unpack WrittenBorder = BlockStart; WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK; } + if (BlockLength <= WriteSize) { var BlockEnd = (BlockStart + BlockLength) & PackDef.MAXWINMASK; @@ -526,6 +567,7 @@ internal sealed partial class Unpack ArrayPool.Shared.Return(FilteredData); FilteredData = ArrayPool.Shared.Rent(FilteredDataSize); } + for (var i = 0; i < FilteredDataSize; i++) { FilteredData[i] = NextPrg.GlobalData[FilteredDataOffset + i]; @@ -558,6 +600,7 @@ internal sealed partial class Unpack filt.NextWindow = false; } } + wrPtr = WrittenBorder; return; } @@ -603,11 +646,13 @@ internal sealed partial class Unpack { return; } + var writeSize = size; if (writeSize > destUnpSize) { writeSize = (int)destUnpSize; } + await writeStream .WriteAsync(data, offset, writeSize, cancellationToken) .ConfigureAwait(false); @@ -786,6 +831,7 @@ internal sealed partial class Unpack NewTable = (BitField & 0x4000) != 0; AddBits(2); } + tablesRead = !NewTable; return !( NewFile || NewTable && !await ReadTablesAsync(cancellationToken).ConfigureAwait(false) @@ -819,9 +865,11 @@ internal sealed partial class Unpack { return false; } + vmCode.Add((byte)(GetBits() >> 8)); AddBits(8); } + return AddVMCode(FirstByte, vmCode); } @@ -831,6 +879,7 @@ internal sealed partial class Unpack { await unpReadBufAsync(cancellationToken).ConfigureAwait(false); } + return InBuf[inAddr++] & 0xff; } @@ -841,6 +890,7 @@ internal sealed partial class Unpack { return false; } + var Length = (FirstByte & 7) + 1; if (Length == 7) { @@ -849,6 +899,7 @@ internal sealed partial class Unpack { return false; } + Length = B1 + 7; } else if (Length == 8) @@ -858,11 +909,13 @@ internal sealed partial class Unpack { return false; } + var B2 = await ppm.DecodeCharAsync(cancellationToken).ConfigureAwait(false); if (B2 == -1) { return false; } + Length = (B1 * 256) + B2; } @@ -874,8 +927,10 @@ internal sealed partial class Unpack { return false; } + vmCode.Add((byte)Ch); } + return AddVMCode(FirstByte, vmCode); } } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index 65697a6a..dcde95d4 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -27,16 +27,22 @@ internal sealed partial class Unpack : BitInput, IRarUnpack if (!disposed) { base.Dispose(); - if (window is not null) - { - ArrayPool.Shared.Return(window); - window = null; - } + ReleaseWindow(); rarVM.Dispose(); + ppm.Dispose(); disposed = true; } } + private void ReleaseWindow() + { + if (window is not null) + { + ArrayPool.Shared.Return(window); + window = null; + } + } + public bool FileExtracted { get; private set; } public long DestSize @@ -61,6 +67,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { unpReadBuf(); } + return (InBuf[inAddr++] & 0xff); } @@ -127,6 +134,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { this.window = ArrayPool.Shared.Rent(PackDef.MAXWINSIZE); } + inAddr = 0; UnpInitData(false); } @@ -139,8 +147,16 @@ internal sealed partial class Unpack : BitInput, IRarUnpack this.writeStream = writeStream; if (!fileHeader.IsSolid) { - Init(); + if (fileHeader.IsStored) + { + ReleaseWindow(); + } + else + { + Init(); + } } + suspended = false; DoUnpack(); } @@ -152,6 +168,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack UnstoreFile(); return; } + switch (fileHeader.CompressionAlgorithm) { case 15: // rar 1.5 compression @@ -189,6 +206,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { break; } + code = code < destUnpSize ? code : (int)destUnpSize; writeStream.Write(buffer.Slice(0, code)); destUnpSize -= code; @@ -227,6 +245,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return; } + if ((!solid || !tablesRead) && !ReadTables()) { return; @@ -259,12 +278,14 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return; } + if (suspended) { FileExtracted = false; return; } } + if (unpBlockType == BlockTypes.BLOCK_PPM) { var Ch = ppm.DecodeChar(); @@ -273,6 +294,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack ppmError = true; break; } + if (Ch == PpmEscChar) { var NextCh = ppm.DecodeChar(); @@ -282,20 +304,25 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { break; } + continue; } + if (NextCh == 2 || NextCh == -1) { break; } + if (NextCh == 3) { if (!ReadVMCodePPM()) { break; } + continue; } + if (NextCh == 4) { int Distance = 0, @@ -322,13 +349,16 @@ internal sealed partial class Unpack : BitInput, IRarUnpack } } } + if (failed) { break; } + CopyString(Length + 32, Distance + 2); continue; } + if (NextCh == 5) { var Length = ppm.DecodeChar(); @@ -336,10 +366,12 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { break; } + CopyString(Length + 4, 1); continue; } } + window[unpPtr++] = (byte)Ch; continue; } @@ -350,6 +382,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack window[unpPtr++] = (byte)Number; continue; } + if (Number >= 271) { var Length = LDecode[Number -= 271] + 3; @@ -370,6 +403,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack Distance += ((Utility.URShift(GetBits(), (20 - Bits))) << 4); AddBits(Bits - 4); } + if (lowDistRepCount > 0) { lowDistRepCount--; @@ -412,30 +446,37 @@ internal sealed partial class Unpack : BitInput, IRarUnpack CopyString(Length, Distance); continue; } + if (Number == 256) { if (!ReadEndOfBlock()) { break; } + continue; } + if (Number == 257) { if (!ReadVMCode()) { break; } + continue; } + if (Number == 258) { if (lastLength != 0) { CopyString(lastLength, lastDist); } + continue; } + if (Number < 263) { var DistNum = Number - 259; @@ -444,6 +485,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { oldDist[I] = oldDist[I - 1]; } + oldDist[0] = Distance; var LengthNumber = this.decodeNumber(RD); @@ -453,10 +495,12 @@ internal sealed partial class Unpack : BitInput, IRarUnpack Length += Utility.URShift(GetBits(), (16 - Bits)); AddBits(Bits); } + InsertLastMatch(Length, Distance); CopyString(Length, Distance); continue; } + if (Number < 272) { var Distance = SDDecode[Number -= 263] + 1; @@ -465,11 +509,13 @@ internal sealed partial class Unpack : BitInput, IRarUnpack Distance += Utility.URShift(GetBits(), (16 - Bits)); AddBits(Bits); } + InsertOldDist(Distance); InsertLastMatch(2, Distance); CopyString(2, Distance); } } + UnpWriteBuf(); } @@ -484,11 +530,13 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { continue; } + if (flt.NextWindow) { flt.NextWindow = false; // ->NextWindow=false; continue; } + var BlockStart = flt.BlockStart; // ->BlockStart; var BlockLength = flt.BlockLength; // ->BlockLength; if (((BlockStart - WrittenBorder) & PackDef.MAXWINMASK) < WriteSize) @@ -499,6 +547,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack WrittenBorder = BlockStart; WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK; } + if (BlockLength <= WriteSize) { var BlockEnd = (BlockStart + BlockLength) & PackDef.MAXWINMASK; @@ -652,6 +701,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack ArrayPool.Shared.Return(FilteredData); FilteredData = ArrayPool.Shared.Rent(FilteredDataSize); } + for (var i = 0; i < FilteredDataSize; i++) { FilteredData[i] = NextPrg.GlobalData[FilteredDataOffset + i]; @@ -682,6 +732,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack filt.NextWindow = false; } } + wrPtr = WrittenBorder; return; } @@ -713,11 +764,13 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return; } + var writeSize = size; if (writeSize > destUnpSize) { writeSize = (int)destUnpSize; } + writeStream.Write(data, offset, writeSize); writtenFileSize += size; @@ -767,6 +820,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { window[unpPtr + i] = window[destPtr + i]; } + unpPtr += length; } else @@ -800,6 +854,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack InitFilters(); } + InitBitInput(); ppmError = false; writtenFileSize = 0; @@ -864,6 +919,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack NewTable = (BitField & 0x4000) != 0; AddBits(2); } + tablesRead = !NewTable; return !(NewFile || NewTable && !ReadTables()); } @@ -880,6 +936,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack return (false); } } + AddBits((8 - inBit) & 7); long bitField = GetBits() & unchecked((int)0xffFFffFF); if ((bitField & 0x8000) != 0) @@ -887,6 +944,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack unpBlockType = BlockTypes.BLOCK_PPM; return (ppm.DecodeInit(this, PpmEscChar)); } + unpBlockType = BlockTypes.BLOCK_LZ; prevLowDist = 0; @@ -896,6 +954,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { new Span(unpOldTable).Clear(); // memset(UnpOldTable,0,sizeof(UnpOldTable)); } + AddBits(2); for (var i = 0; i < PackDef.BC; i++) @@ -917,6 +976,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { bitLength[i++] = 0; } + i--; } } @@ -939,6 +999,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack return (false); } } + var Number = this.decodeNumber(BD); if (Number < 16) { @@ -958,6 +1019,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack N = (Utility.URShift(GetBits(), 9)) + 11; AddBits(7); } + while (N-- > 0 && i < TableSize) { table[i] = table[i - 1]; @@ -977,17 +1039,20 @@ internal sealed partial class Unpack : BitInput, IRarUnpack N = (Utility.URShift(GetBits(), 9)) + 11; AddBits(7); } + while (N-- > 0 && i < TableSize) { table[i++] = 0; } } } + tablesRead = true; if (inAddr > readTop) { return (false); } + UnpackUtility.makeDecodeTables(table, 0, LD, PackDef.NC); UnpackUtility.makeDecodeTables(table, PackDef.NC, DD, PackDef.DC); UnpackUtility.makeDecodeTables(table, PackDef.NC + PackDef.DC, LDD, PackDef.LDC); @@ -1027,9 +1092,11 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + vmCode.Add((byte)(GetBits() >> 8)); AddBits(8); } + return AddVMCode(FirstByte, vmCode); } @@ -1040,6 +1107,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + var Length = (FirstByte & 7) + 1; if (Length == 7) { @@ -1048,6 +1116,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + Length = B1 + 7; } else if (Length == 8) @@ -1057,11 +1126,13 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + var B2 = ppm.DecodeChar(); if (B2 == -1) { return (false); } + Length = (B1 * 256) + B2; } @@ -1073,8 +1144,10 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + vmCode.Add((byte)Ch); // VMCode[I]=Ch; } + return AddVMCode(FirstByte, vmCode); } @@ -1111,6 +1184,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + lastFilter = FiltPos; var NewFilter = (FiltPos == filters.Count); @@ -1151,6 +1225,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { BlockStart += 258; } + StackFilter.BlockStart = ((BlockStart + unpPtr) & PackDef.MAXWINMASK); if ((firstByte & 0x20) != 0) { @@ -1161,6 +1236,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack StackFilter.BlockLength = FiltPos < oldFilterLengths.Count ? oldFilterLengths[FiltPos] : 0; } + StackFilter.NextWindow = (wrPtr != unpPtr) && ((wrPtr - unpPtr) & PackDef.MAXWINMASK) <= BlockStart; @@ -1223,6 +1299,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack ArrayPool.Shared.Return(VMCode); } } + StackFilter.Program.AltCommands = Filter.Program.Commands; // StackFilter->Prg.AltCmd=&Filter->Prg.Cmd[0]; StackFilter.Program.CommandCount = Filter.Program.CommandCount; @@ -1272,6 +1349,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { globalData[0x30 + i] = 0x0; } + if ((firstByte & 8) != 0) // put data block passed as parameter if any { @@ -1279,11 +1357,13 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + var DataSize = RarVM.ReadData(Inp); if (DataSize > RarVM.VM_GLOBALMEMSIZE - RarVM.VM_FIXEDGLOBALSIZE) { return (false); } + var CurSize = StackFilter.Program.GlobalData.Count; if (CurSize < DataSize + RarVM.VM_FIXEDGLOBALSIZE) { @@ -1292,6 +1372,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack DataSize + RarVM.VM_FIXEDGLOBALSIZE - CurSize ); } + var offset = RarVM.VM_FIXEDGLOBALSIZE; globalData = StackFilter.Program.GlobalData; for (var I = 0; I < DataSize; I++) @@ -1300,10 +1381,12 @@ internal sealed partial class Unpack : BitInput, IRarUnpack { return (false); } + globalData[offset + I] = (byte)(Utility.URShift(Inp.GetBits(), 8)); Inp.AddBits(8); } } + return (true); } @@ -1331,10 +1414,6 @@ internal sealed partial class Unpack : BitInput, IRarUnpack private void CleanUp() { - if (ppm != null) - { - var allocator = ppm.SubAlloc; - allocator?.StopSubAllocator(); - } + ppm.Dispose(); } } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index 614e8907..72c808f9 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -59,17 +59,20 @@ internal sealed partial class Unpack : BitInput ArrayPool.Shared.Return(Window); Window = null; } + FragWindow.Reset(); if (FilterSrcMemory.Length != 0) { ArrayPool.Shared.Return(FilterSrcMemory); FilterSrcMemory = Array.Empty(); } + if (FilterDstMemory.Length != 0) { ArrayPool.Shared.Return(FilterDstMemory); FilterDstMemory = Array.Empty(); } + disposed = true; } @@ -120,56 +123,70 @@ internal sealed partial class Unpack : BitInput throw new InvalidFormatException("Grow && Fragmented"); } - var NewWindow = Fragmented ? null : ArrayPool.Shared.Rent((int)WinSize); - - if (NewWindow == null) + byte[] NewWindow = null; + try { - if (Grow || WinSize < 0x1000000) + NewWindow = Fragmented ? null : ArrayPool.Shared.Rent((int)WinSize); + + if (NewWindow == null) { - // We do not support growth for new fragmented window. - // Also exclude RAR4 and small dictionaries. - //throw std::bad_alloc(); - throw new InvalidFormatException("Grow || WinSize<0x1000000"); - } - else - { - if (Window != null) // If allocated by preceding files. + if (Grow || WinSize < 0x1000000) { - //free(Window); - ArrayPool.Shared.Return(Window); - Window = null; + // We do not support growth for new fragmented window. + // Also exclude RAR4 and small dictionaries. + //throw std::bad_alloc(); + throw new InvalidFormatException("Grow || WinSize<0x1000000"); } - FragWindow.Init(WinSize); - Fragmented = true; + else + { + if (Window != null) // If allocated by preceding files. + { + //free(Window); + ArrayPool.Shared.Return(Window); + Window = null; + } + + FragWindow.Init(WinSize); + Fragmented = true; + } + } + + if (!Fragmented) + { + // Clean the window to generate the same output when unpacking corrupt + // RAR files, which may access unused areas of sliding dictionary. + // sharpcompress: don't need this, freshly allocated above + //memset(NewWindow,0,WinSize); + + // If Window is not NULL, it means that window size has grown. + // In solid streams we need to copy data to a new window in such case. + // RAR archiving code does not allow it in solid streams now, + // but let's implement it anyway just in case we'll change it sometimes. + if (Grow) + { + for (size_t I = 1; I <= MaxWinSize; I++) + { + NewWindow[(UnpPtr - I) & (WinSize - 1)] = Window[ + (UnpPtr - I) & (MaxWinSize - 1) + ]; + } + } + + if (Window != null) + { + ArrayPool.Shared.Return(Window); + } + + Window = NewWindow; + NewWindow = null; } } - - if (!Fragmented) + finally { - // Clean the window to generate the same output when unpacking corrupt - // RAR files, which may access unused areas of sliding dictionary. - // sharpcompress: don't need this, freshly allocated above - //memset(NewWindow,0,WinSize); - - // If Window is not NULL, it means that window size has grown. - // In solid streams we need to copy data to a new window in such case. - // RAR archiving code does not allow it in solid streams now, - // but let's implement it anyway just in case we'll change it sometimes. - if (Grow) + if (NewWindow != null) { - for (size_t I = 1; I <= MaxWinSize; I++) - { - NewWindow[(UnpPtr - I) & (WinSize - 1)] = Window[ - (UnpPtr - I) & (MaxWinSize - 1) - ]; - } + ArrayPool.Shared.Return(NewWindow); } - - if (Window != null) - { - ArrayPool.Shared.Return(Window); - } - Window = NewWindow; } MaxWinSize = WinSize; @@ -279,6 +296,7 @@ internal sealed partial class Unpack : BitInput UnpPtr = WrPtr = 0; WriteBorder = Math.Min(MaxWinSize, UNPACK_MAX_WRITE) & MaxWinMask; } + // Filters never share several solid files, so we can safely reset them // even in solid archive. InitFilters(); diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index de2edfd6..13d0ce05 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -321,9 +321,9 @@ "net10.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.6, )", - "resolved": "10.0.6", - "contentHash": "QKuvS0LWX4fjFqeDkyM7Kqt8P3wYTiPD4nwU+9y59n0sCiG714fxDgbbN82vDnzq89AF/PiHl92TP2C4aFDUQA==" + "requested": "[10.0.8, )", + "resolved": "10.0.8", + "contentHash": "dVbSXGIFNR5nZcv2tOLoWI+a9T4jtFd77IYjuND+QVe360qWgAF7H0WtoopYhRw/+SgpGUTyrkrh+65+ClNnfw==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", @@ -441,9 +441,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" + "requested": "[8.0.27, )", + "resolved": "8.0.27", + "contentHash": "rQi9TxifHRnXP7lVRZH05DxD2/XGbJp12q0ozcbrlBlBnyyzssFTH/2vLhtKWUp2CT1qVscTrcYTFiwTyKPKRg==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", diff --git a/tests/SharpCompress.Performance/Program.cs b/tests/SharpCompress.Performance/Program.cs index ab940ed6..81d9d7f3 100644 --- a/tests/SharpCompress.Performance/Program.cs +++ b/tests/SharpCompress.Performance/Program.cs @@ -81,13 +81,13 @@ public class Program using (profiler) { // Run a simple benchmark iteration - var zipBenchmark = new Benchmarks.SevenZipBenchmarks(); + var zipBenchmark = new Benchmarks.RarBenchmarks(); zipBenchmark.Setup(); Console.WriteLine("Running benchmark iterations..."); for (int i = 0; i < 100; i++) { - await zipBenchmark.SevenZipLzma2ExtractAsync_Reader(); + await zipBenchmark.RarExtractArchiveApiAsync(); if (i % 3 == 0) { Console.Write(".");