From 67ceab7443524daa494c4e16257e4c8ae7f79d1b Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 28 May 2026 07:04:28 +0100 Subject: [PATCH] reduce Rar allocations by stackalloc and pooling --- src/SharpCompress/Archives/Rar/RarArchive.cs | 4 ++ .../Rar/RarBLAKE2spStream.Async.cs | 5 +- .../Compressors/Rar/RarBLAKE2spStream.cs | 1 - .../Compressors/Rar/RarCrcStream.Async.cs | 5 +- .../Compressors/Rar/RarCrcStream.cs | 1 - .../Compressors/Rar/RarStream.Async.cs | 24 +++++++++ .../Compressors/Rar/RarStream.cs | 13 +++++ .../Compressors/Rar/UnpackV1/Unpack.Async.cs | 33 +++++++----- .../Rar/UnpackV2017/BitInput.getbits_cpp.cs | 14 ++++- .../Rar/UnpackV2017/BitInput.getbits_hpp.cs | 6 ++- .../FragmentedWindow.unpack50frag_cpp.cs | 8 +-- .../Compressors/Rar/UnpackV2017/Unpack.cs | 46 +++++++++++----- .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 53 ++++++++++++------- src/SharpCompress/Readers/Rar/RarReader.cs | 4 ++ 14 files changed, 159 insertions(+), 58 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchive.cs b/src/SharpCompress/Archives/Rar/RarArchive.cs index deb49aea..14d8aaa3 100644 --- a/src/SharpCompress/Archives/Rar/RarArchive.cs +++ b/src/SharpCompress/Archives/Rar/RarArchive.cs @@ -45,6 +45,10 @@ public partial class RarArchive { unpackV1.Dispose(); } + if (UnpackV2017.IsValueCreated && UnpackV2017.Value is IDisposable unpackV2017) + { + unpackV2017.Dispose(); + } _disposed = true; base.Dispose(); diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.Async.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.Async.cs index 351ad688..53560283 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.Async.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.Async.cs @@ -8,7 +8,7 @@ namespace SharpCompress.Compressors.Rar; internal partial class RarBLAKE2spStream : RarStream { - public static async ValueTask CreateAsync( + public static ValueTask CreateAsync( IRarUnpack unpack, FileHeader fileHeader, MultiVolumeReadOnlyAsyncStream readStream, @@ -16,8 +16,7 @@ internal partial class RarBLAKE2spStream : RarStream ) { var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); - await stream.InitializeAsync(cancellationToken).ConfigureAwait(false); - return stream; + return new ValueTask(stream); } public override async Task ReadAsync( diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs index cbf35b57..e59bfd1b 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -100,7 +100,6 @@ internal partial class RarBLAKE2spStream : RarStream ) { var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); - stream.Initialize(); return stream; } diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs index 670ff8b8..2aedc9a6 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.Async.cs @@ -9,7 +9,7 @@ namespace SharpCompress.Compressors.Rar; internal partial class RarCrcStream : RarStream { - public static async ValueTask CreateAsync( + public static ValueTask CreateAsync( IRarUnpack unpack, FileHeader fileHeader, MultiVolumeReadOnlyStreamBase readStream, @@ -17,8 +17,7 @@ internal partial class RarCrcStream : RarStream ) { var stream = new RarCrcStream(unpack, fileHeader, readStream); - await stream.InitializeAsync(cancellationToken).ConfigureAwait(false); - return stream; + return new ValueTask(stream); } public override async Task ReadAsync( diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 98fbb3b2..a9025e19 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -32,7 +32,6 @@ internal partial class RarCrcStream : RarStream ) { var stream = new RarCrcStream(unpack, fileHeader, readStream); - stream.Initialize(); return stream; } diff --git a/src/SharpCompress/Compressors/Rar/RarStream.Async.cs b/src/SharpCompress/Compressors/Rar/RarStream.Async.cs index 656af1e8..432bd7c5 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.Async.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.Async.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; using System.IO; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; @@ -18,11 +19,17 @@ internal partial class RarStream /// public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { + if (initialized) + { + return; + } + fetch = true; await unpack .DoUnpackAsync(fileHeader, readStream, this, cancellationToken) .ConfigureAwait(false); fetch = false; + initialized = true; _position = 0; } @@ -46,6 +53,12 @@ internal partial class RarStream CancellationToken cancellationToken ) { + if (count == 0) + { + return 0; + } + + await InitializeAsync(cancellationToken).ConfigureAwait(false); outTotal = 0; if (tmpCount > 0) { @@ -87,6 +100,17 @@ internal partial class RarStream ) { cancellationToken.ThrowIfCancellationRequested(); + if (MemoryMarshal.TryGetArray(buffer, out var segment)) + { + return await ReadImplAsync( + segment.Array!, + segment.Offset, + buffer.Length, + cancellationToken + ) + .ConfigureAwait(false); + } + var array = ArrayPool.Shared.Rent(buffer.Length); try { diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index e78faa7a..3604a37b 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -24,6 +24,7 @@ internal partial class RarStream : Stream private int outOffset; private int outCount; private int outTotal; + private bool initialized; private bool isDisposed; private long _position; @@ -36,9 +37,15 @@ internal partial class RarStream : Stream public void Initialize() { + if (initialized) + { + return; + } + fetch = true; unpack.DoUnpack(fileHeader, readStream, this); fetch = false; + initialized = true; _position = 0; } @@ -76,6 +83,12 @@ internal partial class RarStream : Stream public override int Read(byte[] buffer, int offset, int count) { + if (count == 0) + { + return 0; + } + + Initialize(); outTotal = 0; if (tmpCount > 0) { diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs index 48dde9b6..fd986955 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.Async.cs @@ -65,20 +65,29 @@ internal sealed partial class Unpack private async ValueTask UnstoreFileAsync(CancellationToken cancellationToken = default) { - var buffer = new byte[(int)Math.Min(0x10000, destUnpSize)]; - do + var buffer = ArrayPool.Shared.Rent((int)Math.Min(0x10000, destUnpSize)); + try { - var code = await readStream - .ReadAsync(buffer, 0, buffer.Length, cancellationToken) - .ConfigureAwait(false); - if (code == 0 || code == -1) + do { - break; - } - code = code < destUnpSize ? code : (int)destUnpSize; - await writeStream.WriteAsync(buffer, 0, code, cancellationToken).ConfigureAwait(false); - destUnpSize -= code; - } while (!suspended && destUnpSize > 0); + var code = await readStream + .ReadAsync(buffer, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); + if (code == 0 || code == -1) + { + break; + } + code = code < destUnpSize ? code : (int)destUnpSize; + await writeStream + .WriteAsync(buffer, 0, code, cancellationToken) + .ConfigureAwait(false); + destUnpSize -= code; + } while (!suspended && destUnpSize > 0); + } + finally + { + ArrayPool.Shared.Return(buffer); + } } private async ValueTask Unpack29Async(bool solid, CancellationToken cancellationToken = default) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_cpp.cs index 8fed421c..8281ee01 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_cpp.cs @@ -1,4 +1,5 @@ -using size_t = System.UInt32; +using System.Buffers; +using size_t = System.UInt32; #nullable disable @@ -16,7 +17,7 @@ internal partial class BitInput // read only 1 byte from the last position of buffer and avoid a crash // from access to next 3 bytes, which contents we do not need. size_t BufSize = MAX_SIZE + 3; - InBuf = new byte[BufSize]; + InBuf = ArrayPool.Shared.Rent(checked((int)BufSize)); // Ensure that we get predictable results when accessing bytes in area // not filled with read data. @@ -42,4 +43,13 @@ internal partial class BitInput public uint fgetbits() => // Function wrapped version of inline getbits to save code size. getbits(); + + public virtual void Dispose() + { + if (!ExternalBuffer && InBuf != null) + { + ArrayPool.Shared.Return(InBuf); + InBuf = null; + } + } } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_hpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_hpp.cs index f245ea19..b3fad1ad 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_hpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/BitInput.getbits_hpp.cs @@ -1,6 +1,8 @@ -namespace SharpCompress.Compressors.Rar.UnpackV2017; +using System; -internal partial class BitInput +namespace SharpCompress.Compressors.Rar.UnpackV2017; + +internal partial class BitInput : IDisposable { public const int MAX_SIZE = 0x8000; // Size of input buffer. diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/FragmentedWindow.unpack50frag_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/FragmentedWindow.unpack50frag_cpp.cs index f269fb5f..ff1b4631 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/FragmentedWindow.unpack50frag_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/FragmentedWindow.unpack50frag_cpp.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using SharpCompress.Common; using size_t = System.UInt32; @@ -19,15 +20,16 @@ internal partial class FragmentedWindow // Reset(); //} - private void Reset() + public void Reset() { for (uint I = 0; I < Mem.Length; I++) { if (Mem[I] != null) { - //free(Mem[I]); + ArrayPool.Shared.Return(Mem[I]); Mem[I] = null; } + MemSize[I] = 0; } } @@ -50,7 +52,7 @@ internal partial class FragmentedWindow byte[] NewMem = null; while (Size >= MinSize) { - NewMem = new byte[Size]; + NewMem = ArrayPool.Shared.Rent(checked((int)Size)); if (NewMem != null) { break; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index e18e33d6..920d7fa5 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -135,19 +136,26 @@ internal partial class Unpack : IRarUnpack private async ValueTask UnstoreFileAsync(CancellationToken cancellationToken = default) { - var buffer = new byte[(int)Math.Min(0x10000, DestUnpSize)]; - do + var buffer = ArrayPool.Shared.Rent((int)Math.Min(0x10000, DestUnpSize)); + try { - var n = await readStream - .ReadAsync(buffer, 0, buffer.Length, cancellationToken) - .ConfigureAwait(false); - if (n == 0) + do { - break; - } - await writeStream.WriteAsync(buffer, 0, n, cancellationToken).ConfigureAwait(false); - DestUnpSize -= n; - } while (!Suspended); + var n = await readStream + .ReadAsync(buffer, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); + if (n == 0) + { + break; + } + await writeStream.WriteAsync(buffer, 0, n, cancellationToken).ConfigureAwait(false); + DestUnpSize -= n; + } while (!Suspended); + } + finally + { + ArrayPool.Shared.Return(buffer); + } } public bool Suspended { get; set; } @@ -181,6 +189,18 @@ internal partial class Unpack : IRarUnpack set => PPMEscChar = value; } - public static byte[] EnsureCapacity(byte[] array, int length) => - array.Length < length ? new byte[length] : array; + private byte[] EnsureCapacity(byte[] array, int length) + { + if (array.Length >= length) + { + return array; + } + + var newArray = ArrayPool.Shared.Rent(length); + if (array.Length != 0) + { + ArrayPool.Shared.Return(array); + } + return newArray; + } } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index d244ebb8..614e8907 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -13,6 +13,8 @@ namespace SharpCompress.Compressors.Rar.UnpackV2017; internal sealed partial class Unpack : BitInput { + private bool disposed; + public Unpack( /* ComprDataIO *DataIO */ ) //:Inp(true),VMCodeInp(true) @@ -44,19 +46,32 @@ internal sealed partial class Unpack : BitInput InitHuff(); } - // later: may need Dispose() if we support thread pool - //Unpack::~Unpack() - //{ - // InitFilters30(false); - // - // if (Window!=null) - // free(Window); - //#if RarV2017_RAR_SMP - // DestroyThreadPool(UnpThreadPool); - // delete[] ReadBufMT; - // delete[] UnpThreadData; - //#endif - //} + public override void Dispose() + { + if (disposed) + { + return; + } + + base.Dispose(); + if (Window != null) + { + ArrayPool.Shared.Return(Window); + Window = null; + } + FragWindow.Reset(); + if (FilterSrcMemory.Length != 0) + { + ArrayPool.Shared.Return(FilterSrcMemory); + FilterSrcMemory = Array.Empty(); + } + if (FilterDstMemory.Length != 0) + { + ArrayPool.Shared.Return(FilterDstMemory); + FilterDstMemory = Array.Empty(); + } + disposed = true; + } private void Init(size_t WinSize, bool Solid) { @@ -150,8 +165,10 @@ internal sealed partial class Unpack : BitInput } } - //if (Window!=null) - // free(Window); + if (Window != null) + { + ArrayPool.Shared.Return(Window); + } Window = NewWindow; } @@ -290,7 +307,7 @@ internal sealed partial class Unpack : BitInput Dec.MaxNum = Size; // Calculate how many entries for every bit length in LengthTable we have. - var LengthCount = new uint[16]; + Span LengthCount = stackalloc uint[16]; //memset(LengthCount,0,sizeof(LengthCount)); for (size_t I = 0; I < Size; I++) { @@ -334,9 +351,9 @@ internal sealed partial class Unpack : BitInput // Prepare the copy of DecodePos. We'll modify this copy below, // so we cannot use the original DecodePos. - var CopyDecodePos = new uint[Dec.DecodePos.Length]; + Span CopyDecodePos = stackalloc uint[16]; //memcpy(CopyDecodePos,Dec->DecodePos,sizeof(CopyDecodePos)); - Array.Copy(Dec.DecodePos, CopyDecodePos, CopyDecodePos.Length); + Dec.DecodePos.AsSpan().CopyTo(CopyDecodePos); // For every bit length in the bit length table and so for every item // of alphabet. diff --git a/src/SharpCompress/Readers/Rar/RarReader.cs b/src/SharpCompress/Readers/Rar/RarReader.cs index 2c34196b..829cf97a 100644 --- a/src/SharpCompress/Readers/Rar/RarReader.cs +++ b/src/SharpCompress/Readers/Rar/RarReader.cs @@ -31,6 +31,10 @@ public abstract partial class RarReader : AbstractReader