From e30a88e634a3d10e02b145638eccb24eadb0cd2e Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 17 Oct 2025 10:50:09 +0100 Subject: [PATCH] Started rar async...interface changes were viral --- .../Archives/GZip/GZipArchiveEntry.cs | 3 + src/SharpCompress/Archives/IArchiveEntry.cs | 6 + .../Archives/Rar/RarArchiveEntry.cs | 10 +- .../Archives/SevenZip/SevenZipArchiveEntry.cs | 2 + .../Archives/Tar/TarArchiveEntry.cs | 2 + .../Archives/Zip/ZipArchiveEntry.cs | 2 + src/SharpCompress/Common/EntryStream.cs | 11 +- .../Compressors/Rar/IRarUnpack.cs | 7 +- .../Compressors/Rar/RarBLAKE2spStream.cs | 10 +- .../Compressors/Rar/RarStream.cs | 217 ++++++++- .../Compressors/Rar/UnpackV1/Unpack.cs | 15 + .../Compressors/Rar/UnpackV2017/Unpack.cs | 3 +- .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 44 +- .../UnpackV2017/Unpack.unpack_cpp_async.cs | 410 ++++++++++++++++++ .../Rar/UnpackV2017/Unpack_async.cs | 114 +++++ src/SharpCompress/Readers/AbstractReader.cs | 35 +- src/SharpCompress/Readers/IReader.cs | 7 +- src/SharpCompress/Readers/Rar/RarReader.cs | 12 +- src/SharpCompress/Utility.cs | 14 + 19 files changed, 860 insertions(+), 64 deletions(-) create mode 100644 src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp_async.cs create mode 100644 src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack_async.cs diff --git a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs index f00e889c..1d84b481 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs @@ -1,5 +1,6 @@ using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common.GZip; namespace SharpCompress.Archives.GZip; @@ -20,6 +21,8 @@ public class GZipArchiveEntry : GZipEntry, IArchiveEntry return Parts.Single().GetCompressedStream().NotNull(); } + public virtual async Task OpenEntryStreamAsync() => await Task.FromResult(OpenEntryStream()); + #region IArchiveEntry Members public IArchive Archive { get; } diff --git a/src/SharpCompress/Archives/IArchiveEntry.cs b/src/SharpCompress/Archives/IArchiveEntry.cs index 708753cb..58b8a511 100644 --- a/src/SharpCompress/Archives/IArchiveEntry.cs +++ b/src/SharpCompress/Archives/IArchiveEntry.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Archives; @@ -11,6 +12,11 @@ public interface IArchiveEntry : IEntry /// Stream OpenEntryStream(); + /// + /// Opens the current entry as a stream that will decompress as it is read. + /// Read the entire stream or use SkipEntry on EntryStream. + /// + Task OpenEntryStreamAsync(); /// /// The archive can find all the parts of the archive needed to extract this entry. /// diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 262d7cbe..b8bce402 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar; using SharpCompress.Common.Rar.Headers; @@ -66,18 +67,21 @@ public class RarArchiveEntry : RarEntry, IArchiveEntry } } - public Stream OpenEntryStream() + public Stream OpenEntryStream() => throw new NotSupportedException("Synchronous extraction is not supported. Use OpenEntryStreamAsync instead."); + + + public async Task OpenEntryStreamAsync() { if (IsRarV3) { - return new RarStream( + return await RarStream.Create( archive.UnpackV1.Value, FileHeader, new MultiVolumeReadOnlyStream(Parts.Cast(), archive) ); } - return new RarStream( + return await RarStream.Create( archive.UnpackV2017.Value, FileHeader, new MultiVolumeReadOnlyStream(Parts.Cast(), archive) diff --git a/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs b/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs index 9824ca47..7818c29f 100644 --- a/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Threading.Tasks; using SharpCompress.Common.SevenZip; namespace SharpCompress.Archives.SevenZip; @@ -9,6 +10,7 @@ public class SevenZipArchiveEntry : SevenZipEntry, IArchiveEntry : base(part) => Archive = archive; public Stream OpenEntryStream() => FilePart.GetCompressedStream(); + public virtual async Task OpenEntryStreamAsync() => await Task.FromResult(OpenEntryStream()); public IArchive Archive { get; } diff --git a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs index 770a7109..64b382a7 100644 --- a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs @@ -1,5 +1,6 @@ using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Tar; @@ -11,6 +12,7 @@ public class TarArchiveEntry : TarEntry, IArchiveEntry : base(part, compressionType) => Archive = archive; public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); + public virtual async Task OpenEntryStreamAsync() => await Task.FromResult(OpenEntryStream()); #region IArchiveEntry Members diff --git a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs index 4c980f91..74ce021c 100644 --- a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs @@ -1,5 +1,6 @@ using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common.Zip; namespace SharpCompress.Archives.Zip; @@ -10,6 +11,7 @@ public class ZipArchiveEntry : ZipEntry, IArchiveEntry : base(part) => Archive = archive; public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); + public virtual async Task OpenEntryStreamAsync() => await Task.FromResult(OpenEntryStream()); #region IArchiveEntry Members diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index a0fe736a..60190289 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Compression; +using System.Threading.Tasks; using SharpCompress.IO; using SharpCompress.Readers; @@ -47,10 +48,18 @@ public class EntryStream : Stream, IStreamStack /// public void SkipEntry() { - this.Skip(); + this.Skip(); _completed = true; } + /// + /// When reading a stream from OpenEntryStream, the stream must be completed so use this to finish reading the entire entry. + /// + public async Task SkipEntryAsync() + { + await this.SkipAsync(); + _completed = true; + } protected override void Dispose(bool disposing) { if (!(_completed || _reader.Cancelled)) diff --git a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs index 651e2c8a..3d7cd615 100644 --- a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs +++ b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs @@ -1,13 +1,18 @@ using System.IO; +using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; namespace SharpCompress.Compressors.Rar; internal interface IRarUnpack { + #if NETSTANDARD2_0 || NETFRAMEWORK void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStream); void DoUnpack(); - +#else + ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream); + ValueTask DoUnpackAsync(); +#endif // eg u/i pause/resume button bool Suspended { get; set; } diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs index c0aead0d..0f69fdb5 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; using SharpCompress.IO; @@ -9,6 +10,13 @@ namespace SharpCompress.Compressors.Rar; internal class RarBLAKE2spStream : RarStream, IStreamStack { + public static async ValueTask Create(IRarUnpack unpack, FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream) + { + var rs = new RarBLAKE2spStream(unpack, fileHeader, readStream); + await RarStream.Initialize(rs, unpack, fileHeader, readStream); + return rs; + } #if DEBUG_STREAMS long IStreamStack.InstanceId { get; set; } #endif @@ -103,7 +111,7 @@ internal class RarBLAKE2spStream : RarStream, IStreamStack byte[] _hash = { }; - public RarBLAKE2spStream( + protected RarBLAKE2spStream( IRarUnpack unpack, FileHeader fileHeader, MultiVolumeReadOnlyStream readStream diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index f2f4b219..d5a08aca 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -1,8 +1,10 @@ -#nullable disable + using System; using System.Buffers; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; using SharpCompress.IO; @@ -36,18 +38,40 @@ internal class RarStream : Stream, IStreamStack private bool fetch; - private byte[] tmpBuffer = ArrayPool.Shared.Rent(65536); + private byte[]? tmpBuffer = ArrayPool.Shared.Rent(65536); private int tmpOffset; private int tmpCount; - private byte[] outBuffer; + private byte[]? outBuffer; private int outOffset; private int outCount; private int outTotal; private bool isDisposed; private long _position; - public RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream) + public static async ValueTask Create(IRarUnpack unpack, FileHeader fileHeader, Stream readStream) + { + var rs = new RarStream(unpack, fileHeader, readStream); + await Initialize(rs, unpack, fileHeader, readStream); + return rs; + } + + internal static async ValueTask Initialize(RarStream rs,IRarUnpack unpack, FileHeader fileHeader, Stream readStream) + { + + + rs.fetch = true; +#if !NETSTANDARD2_0 && !NETFRAMEWORK + await unpack.DoUnpackAsync(fileHeader, readStream, rs); +#else + unpack.DoUnpack(fileHeader, readStream, rs); + await Task.CompletedTask; +#endif + rs.fetch = false; + rs._position = 0; + } + + protected RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream) { this.unpack = unpack; this.fileHeader = fileHeader; @@ -56,11 +80,6 @@ internal class RarStream : Stream, IStreamStack #if DEBUG_STREAMS this.DebugConstruct(typeof(RarStream)); #endif - - fetch = true; - unpack.DoUnpack(fileHeader, readStream, this); - fetch = false; - _position = 0; } protected override void Dispose(bool disposing) @@ -72,14 +91,36 @@ internal class RarStream : Stream, IStreamStack #if DEBUG_STREAMS this.DebugDispose(typeof(RarStream)); #endif - ArrayPool.Shared.Return(this.tmpBuffer); - this.tmpBuffer = null; + if (tmpBuffer != null) + { + ArrayPool.Shared.Return(this.tmpBuffer); + this.tmpBuffer = null; + } } isDisposed = true; base.Dispose(disposing); readStream.Dispose(); } } +#if !NETSTANDARD2_0 && !NETFRAMEWORK + public override async ValueTask DisposeAsync() + { + if (!isDisposed) + { +#if DEBUG_STREAMS + this.DebugDispose(typeof(RarStream)); +#endif + if (tmpBuffer != null) + { + ArrayPool.Shared.Return(this.tmpBuffer); + this.tmpBuffer = null; + } + isDisposed = true; + await readStream.DisposeAsync().ConfigureAwait(false); + } + await base.DisposeAsync().ConfigureAwait(false); + } +#endif public override bool CanRead => true; @@ -89,6 +130,8 @@ internal class RarStream : Stream, IStreamStack public override void Flush() { } + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public override long Length => fileHeader.UncompressedSize; //commented out code always returned the length of the file @@ -98,8 +141,87 @@ internal class RarStream : Stream, IStreamStack set => throw new NotSupportedException(); } +#if !NETSTANDARD2_0 && !NETFRAMEWORK + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var bytesRead = Read(buffer, offset, count); + return Task.FromResult(bytesRead); + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + outTotal = 0; + var count = buffer.Length; + var offset = 0; + + if (tmpCount > 0) + { + var toCopy = tmpCount < count ? tmpCount : count; + tmpBuffer.AsSpan(tmpOffset, toCopy).CopyTo(buffer.Span.Slice(offset, toCopy)); + tmpOffset += toCopy; + tmpCount -= toCopy; + offset += toCopy; + count -= toCopy; + outTotal += toCopy; + } + if (count > 0 && unpack.DestSize > 0) + { + // Create a temporary array for the unpack operation + var tempArray = ArrayPool.Shared.Rent(count); + try + { + outBuffer = tempArray; + outOffset = 0; + outCount = count; + fetch = true; + await unpack.DoUnpackAsync(); + fetch = false; + + // Copy the unpacked data to the memory buffer + var unpacked = outTotal - (tmpCount > 0 ? offset : 0); + if (unpacked > 0) + { + tempArray.AsSpan(0, unpacked).CopyTo(buffer.Span.Slice(offset, unpacked)); + } + } + finally + { + ArrayPool.Shared.Return(tempArray); + outBuffer = null; + } + } + _position += outTotal; + if (count > 0 && outTotal == 0 && _position != Length) + { + // sanity check, eg if we try to decompress a redir entry + throw new InvalidOperationException( + $"unpacked file size does not match header: expected {Length} found {_position}" + ); + } + return outTotal; + } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException("Use ReadAsync or ReadAsync(Memory) instead."); +#else + + public override int Read(byte[] buffer, int offset, int count) { + if (tmpBuffer == null) + { + throw new ObjectDisposedException(nameof(RarStream)); + } outTotal = 0; if (tmpCount > 0) { @@ -130,6 +252,7 @@ internal class RarStream : Stream, IStreamStack } return outTotal; } +#endif public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); @@ -137,6 +260,14 @@ internal class RarStream : Stream, IStreamStack public override void Write(byte[] buffer, int offset, int count) { + if (tmpBuffer == null) + { + throw new ObjectDisposedException(nameof(RarStream)); + } + if (outBuffer == null) + { + throw new ObjectDisposedException(nameof(RarStream)); + } if (!fetch) { throw new NotSupportedException(); @@ -165,8 +296,72 @@ internal class RarStream : Stream, IStreamStack } } + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + Write(buffer, offset, count); + return Task.CompletedTask; + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } +#if !NETSTANDARD2_0 && !NETFRAMEWORK + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + if (!fetch) + { + throw new NotSupportedException(); + } + + var count = buffer.Length; + var offset = 0; + + if (outCount > 0) + { + var toCopy = outCount < count ? outCount : count; + buffer.Span.Slice(offset, toCopy).CopyTo(outBuffer.AsSpan(outOffset, toCopy)); + outOffset += toCopy; + outCount -= toCopy; + offset += toCopy; + count -= toCopy; + outTotal += toCopy; + } + if (count > 0) + { + EnsureBufferCapacity(count); + buffer.Span.Slice(offset, count).CopyTo(tmpBuffer.AsSpan(tmpCount, count)); + tmpCount += count; + tmpOffset = 0; + unpack.Suspended = true; + } + else + { + unpack.Suspended = false; + } + return ValueTask.CompletedTask; + } + catch (Exception ex) + { + return new ValueTask(Task.FromException(ex)); + } + } +#endif + private void EnsureBufferCapacity(int count) { + if (tmpBuffer == null) + { + throw new ObjectDisposedException(nameof(RarStream)); + } if (this.tmpBuffer.Length < this.tmpCount + count) { var newLength = diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index c4e6d108..f783a645 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -4,6 +4,7 @@ using System; using System.Buffers; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; using SharpCompress.Compressors.PPMd.H; @@ -153,6 +154,20 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable DoUnpack(); } +#if !NETSTANDARD2_0 && !NETFRAMEWORK + public ValueTask DoUnpackAsync() + { + DoUnpack(); + return ValueTask.CompletedTask; + } + + public ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream) + { + DoUnpack(fileHeader, readStream, writeStream); + return ValueTask.CompletedTask; + } +#endif + public void DoUnpack() { if (fileHeader.CompressionMethod == 0) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index 22fdc746..922c31de 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -1,3 +1,4 @@ +#if NETSTANDARD2_0 || NETFRAMEWORK using System; using System.IO; using SharpCompress.Common.Rar.Headers; @@ -64,7 +65,6 @@ internal partial class Unpack : IRarUnpack DoUnpack(fileHeader.CompressionAlgorithm, fileHeader.IsSolid); } } - private void UnstoreFile() { Span b = stackalloc byte[(int)Math.Min(0x10000, DestUnpSize)]; @@ -106,3 +106,4 @@ internal partial class Unpack : IRarUnpack public static byte[] EnsureCapacity(byte[] array, int length) => array.Length < length ? new byte[length] : array; } +#endif diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index 92a37c8b..a65d7178 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -1,3 +1,4 @@ +#if NETSTANDARD2_0 || NETFRAMEWORK #nullable disable using System; @@ -29,12 +30,12 @@ internal sealed partial class Unpack : BitInput Suspended = false; UnpAllBuf = false; UnpSomeRead = false; - /*#if RarV2017_RAR_SMP - MaxUserThreads = 1; - UnpThreadPool = CreateThreadPool(); - ReadBufMT = null; - UnpThreadData = null; - #endif*/ + // #if RarV2017_RAR_SMP + // MaxUserThreads = 1; + // UnpThreadPool = CreateThreadPool(); + // ReadBufMT = null; + // UnpThreadData = null; + // #endif MaxWinSize = 0; MaxWinMask = 0; @@ -197,21 +198,21 @@ internal sealed partial class Unpack : BitInput break; #endif case 50: // RAR 5.0 compression algorithm. - /*#if RarV2017_RAR_SMP - if (MaxUserThreads > 1) - { - // We do not use the multithreaded unpack routine to repack RAR archives - // in 'suspended' mode, because unlike the single threaded code it can - // write more than one dictionary for same loop pass. So we would need - // larger buffers of unknown size. Also we do not support multithreading - // in fragmented window mode. - if (!Fragmented) - { - Unpack5MT(Solid); - break; - } - } - #endif*/ + // #if RarV2017_RAR_SMP + // if (MaxUserThreads > 1) + // { + // // We do not use the multithreaded unpack routine to repack RAR archives + // // in 'suspended' mode, because unlike the single threaded code it can + // // write more than one dictionary for same loop pass. So we would need + // // larger buffers of unknown size. Also we do not support multithreading + // // in fragmented window mode. + // if (!Fragmented) + // { + // Unpack5MT(Solid); + // break; + // } + // } + // #endif Unpack5(Solid); break; #if !Rar2017_NOSTRICT @@ -407,3 +408,4 @@ internal sealed partial class Unpack : BitInput } } } +#endif diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp_async.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp_async.cs new file mode 100644 index 00000000..2cdec365 --- /dev/null +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp_async.cs @@ -0,0 +1,410 @@ +#if !NETSTANDARD2_0 && !NETFRAMEWORK + +using System; +using SharpCompress.Common; +using static SharpCompress.Compressors.Rar.UnpackV2017.PackDef; +using static SharpCompress.Compressors.Rar.UnpackV2017.UnpackGlobal; +#if !Rar2017_64bit +using size_t = System.UInt32; +#else +using nint = System.Int64; +using nuint = System.UInt64; +using size_t = System.UInt64; +#endif + +namespace SharpCompress.Compressors.Rar.UnpackV2017; + +internal sealed partial class Unpack : BitInput +{ + public Unpack( /* ComprDataIO *DataIO */ + ) + //:Inp(true),VMCodeInp(true) + : base(true) + { + _UnpackCtor(); + + //UnpIO=DataIO; + Window = null; + Fragmented = false; + Suspended = false; + UnpAllBuf = false; + UnpSomeRead = false; + // #if RarV2017_RAR_SMP + // MaxUserThreads = 1; + // UnpThreadPool = CreateThreadPool(); + // ReadBufMT = null; + // UnpThreadData = null; + // #endif + MaxWinSize = 0; + MaxWinMask = 0; + + // Perform initialization, which should be done only once for all files. + // It prevents crash if first DoUnpack call is later made with wrong + // (true) 'Solid' value. + UnpInitData(false); +#if !RarV2017_SFX_MODULE + // RAR 1.5 decompression initialization + UnpInitData15(false); + InitHuff(); +#endif + } + + // later: may need Dispose() if we support thread pool + //Unpack::~Unpack() + //{ + // InitFilters30(false); + // + // if (Window!=null) + // free(Window); + //#if RarV2017_RAR_SMP + // DestroyThreadPool(UnpThreadPool); + // delete[] ReadBufMT; + // delete[] UnpThreadData; + //#endif + //} + + private void Init(size_t WinSize, bool Solid) + { + // If 32-bit RAR unpacks an archive with 4 GB dictionary, the window size + // will be 0 because of size_t overflow. Let's issue the memory error. + if (WinSize == 0) + //ErrHandler.MemoryError(); + { + throw new InvalidFormatException( + "invalid window size (possibly due to a rar file with a 4GB being unpacked on a 32-bit platform)" + ); + } + + // Minimum window size must be at least twice more than maximum possible + // size of filter block, which is 0x10000 in RAR now. If window size is + // smaller, we can have a block with never cleared flt->NextWindow flag + // in UnpWriteBuf(). Minimum window size 0x20000 would be enough, but let's + // use 0x40000 for extra safety and possible filter area size expansion. + const size_t MinAllocSize = 0x40000; + if (WinSize < MinAllocSize) + { + WinSize = MinAllocSize; + } + + if (WinSize <= MaxWinSize) // Use the already allocated window. + { + return; + } + + if ((WinSize >> 16) > 0x10000) // Window size must not exceed 4 GB. + { + return; + } + + // Archiving code guarantees that window size does not grow in the same + // solid stream. So if we are here, we are either creating a new window + // or increasing the size of non-solid window. So we could safely reject + // current window data without copying them to a new window, though being + // extra cautious, we still handle the solid window grow case below. + var Grow = Solid && (Window != null || Fragmented); + + // We do not handle growth for existing fragmented window. + if (Grow && Fragmented) + //throw std::bad_alloc(); + { + throw new InvalidFormatException("Grow && Fragmented"); + } + + var NewWindow = Fragmented ? null : new byte[WinSize]; + + if (NewWindow == null) + { + if (Grow || WinSize < 0x1000000) + { + // We do not support growth for new fragmented window. + // Also exclude RAR4 and small dictionaries. + //throw std::bad_alloc(); + throw new InvalidFormatException("Grow || WinSize<0x1000000"); + } + else + { + if (Window != null) // If allocated by preceding files. + { + //free(Window); + Window = null; + } + FragWindow.Init(WinSize); + Fragmented = true; + } + } + + if (!Fragmented) + { + // Clean the window to generate the same output when unpacking corrupt + // RAR files, which may access unused areas of sliding dictionary. + // sharpcompress: don't need this, freshly allocated above + //memset(NewWindow,0,WinSize); + + // If Window is not NULL, it means that window size has grown. + // In solid streams we need to copy data to a new window in such case. + // RAR archiving code does not allow it in solid streams now, + // but let's implement it anyway just in case we'll change it sometimes. + if (Grow) + { + for (size_t I = 1; I <= MaxWinSize; I++) + { + NewWindow[(UnpPtr - I) & (WinSize - 1)] = Window[ + (UnpPtr - I) & (MaxWinSize - 1) + ]; + } + } + + //if (Window!=null) + // free(Window); + Window = NewWindow; + } + + MaxWinSize = WinSize; + MaxWinMask = MaxWinSize - 1; + } + + private void DoUnpack(uint Method, bool Solid) + { + // Methods <50 will crash in Fragmented mode when accessing NULL Window. + // They cannot be called in such mode now, but we check it below anyway + // just for extra safety. + switch (Method) + { +#if !RarV2017_SFX_MODULE + case 15: // rar 1.5 compression + if (!Fragmented) + { + Unpack15(Solid); + } + + break; + case 20: // rar 2.x compression + case 26: // files larger than 2GB + if (!Fragmented) + { + Unpack20(Solid); + } + + break; +#endif +#if !RarV2017_RAR5ONLY + case 29: // rar 3.x compression + if (!Fragmented) + { + throw new NotImplementedException(); + } + + break; +#endif + case 50: // RAR 5.0 compression algorithm. + // #if RarV2017_RAR_SMP + // if (MaxUserThreads > 1) + // { + // // We do not use the multithreaded unpack routine to repack RAR archives + // // in 'suspended' mode, because unlike the single threaded code it can + // // write more than one dictionary for same loop pass. So we would need + // // larger buffers of unknown size. Also we do not support multithreading + // // in fragmented window mode. + // if (!Fragmented) + // { + // Unpack5MT(Solid); + // break; + // } + // } + // #endif + Unpack5(Solid); + break; +#if !Rar2017_NOSTRICT + default: + throw new InvalidFormatException("unknown compression method " + Method); +#endif + } + } + + private void UnpInitData(bool Solid) + { + if (!Solid) + { + new Span(OldDist).Clear(); + OldDistPtr = 0; + LastDist = LastLength = 0; + // memset(Window,0,MaxWinSize); + //memset(&BlockTables,0,sizeof(BlockTables)); + BlockTables = new UnpackBlockTables(); + // sharpcompress: no default ctor for struct + BlockTables.Init(); + UnpPtr = WrPtr = 0; + WriteBorder = Math.Min(MaxWinSize, UNPACK_MAX_WRITE) & MaxWinMask; + } + // Filters never share several solid files, so we can safely reset them + // even in solid archive. + InitFilters(); + + Inp.InitBitInput(); + WrittenFileSize = 0; + ReadTop = 0; + ReadBorder = 0; + + //memset(&BlockHeader,0,sizeof(BlockHeader)); + BlockHeader = new UnpackBlockHeader(); + BlockHeader.BlockSize = -1; // '-1' means not defined yet. +#if !RarV2017_SFX_MODULE + UnpInitData20(Solid); +#endif + //UnpInitData30(Solid); + UnpInitData50(Solid); + } + + // LengthTable contains the length in bits for every element of alphabet. + // Dec is the structure to decode Huffman code/ + // Size is size of length table and DecodeNum field in Dec structure, + private void MakeDecodeTables(Span LengthTable, int offset, DecodeTable Dec, uint Size) + { + // Size of alphabet and DecodePos array. + Dec.MaxNum = Size; + + // Calculate how many entries for every bit length in LengthTable we have. + var LengthCount = new uint[16]; + //memset(LengthCount,0,sizeof(LengthCount)); + for (size_t I = 0; I < Size; I++) + { + LengthCount[LengthTable[checked((int)(offset + I))] & 0xf]++; + } + + // We must not calculate the number of zero length codes. + LengthCount[0] = 0; + + // Set the entire DecodeNum to zero. + //memset(Dec->DecodeNum,0,Size*sizeof(*Dec->DecodeNum)); + new Span(Dec.DecodeNum).Clear(); + + // Initialize not really used entry for zero length code. + Dec.DecodePos[0] = 0; + + // Start code for bit length 1 is 0. + Dec.DecodeLen[0] = 0; + + // Right aligned upper limit code for current bit length. + uint UpperLimit = 0; + + for (var I = 1; I < 16; I++) + { + // Adjust the upper limit code. + UpperLimit += LengthCount[I]; + + // Left aligned upper limit code. + var LeftAligned = UpperLimit << (16 - I); + + // Prepare the upper limit code for next bit length. + UpperLimit *= 2; + + // Store the left aligned upper limit code. + Dec.DecodeLen[I] = LeftAligned; + + // Every item of this array contains the sum of all preceding items. + // So it contains the start position in code list for every bit length. + Dec.DecodePos[I] = Dec.DecodePos[I - 1] + LengthCount[I - 1]; + } + + // Prepare the copy of DecodePos. We'll modify this copy below, + // so we cannot use the original DecodePos. + var CopyDecodePos = new uint[Dec.DecodePos.Length]; + //memcpy(CopyDecodePos,Dec->DecodePos,sizeof(CopyDecodePos)); + Array.Copy(Dec.DecodePos, CopyDecodePos, CopyDecodePos.Length); + + // For every bit length in the bit length table and so for every item + // of alphabet. + for (uint I = 0; I < Size; I++) + { + // Get the current bit length. + var _CurBitLength = (byte)(LengthTable[checked((int)(offset + I))] & 0xf); + + if (_CurBitLength != 0) + { + // Last position in code list for current bit length. + var LastPos = CopyDecodePos[_CurBitLength]; + + // Prepare the decode table, so this position in code list will be + // decoded to current alphabet item number. + Dec.DecodeNum[LastPos] = (ushort)I; + + // We'll use next position number for this bit length next time. + // So we pass through the entire range of positions available + // for every bit length. + CopyDecodePos[_CurBitLength]++; + } + } + + // Define the number of bits to process in quick mode. We use more bits + // for larger alphabets. More bits means that more codes will be processed + // in quick mode, but also that more time will be spent to preparation + // of tables for quick decode. + switch (Size) + { + case NC: + case NC20: + case NC30: + Dec.QuickBits = MAX_QUICK_DECODE_BITS; + break; + default: + Dec.QuickBits = MAX_QUICK_DECODE_BITS - 3; + break; + } + + // Size of tables for quick mode. + var QuickDataSize = 1U << (int)Dec.QuickBits; + + // Bit length for current code, start from 1 bit codes. It is important + // to use 1 bit instead of 0 for minimum code length, so we are moving + // forward even when processing a corrupt archive. + //uint CurBitLength=1; + byte CurBitLength = 1; + + // For every right aligned bit string which supports the quick decoding. + for (uint Code = 0; Code < QuickDataSize; Code++) + { + // Left align the current code, so it will be in usual bit field format. + var BitField = Code << (int)(16 - Dec.QuickBits); + + // Prepare the table for quick decoding of bit lengths. + + // Find the upper limit for current bit field and adjust the bit length + // accordingly if necessary. + while (CurBitLength < Dec.DecodeLen.Length && BitField >= Dec.DecodeLen[CurBitLength]) + { + CurBitLength++; + } + + // Translation of right aligned bit string to bit length. + Dec.QuickLen[Code] = CurBitLength; + + // Prepare the table for quick translation of position in code list + // to position in alphabet. + + // Calculate the distance from the start code for current bit length. + var Dist = BitField - Dec.DecodeLen[CurBitLength - 1]; + + // Right align the distance. + Dist >>= (16 - CurBitLength); + + // Now we can calculate the position in the code list. It is the sum + // of first position for current bit length and right aligned distance + // between our bit field and start code for current bit length. + uint Pos; + if ( + CurBitLength < Dec.DecodePos.Length + && (Pos = Dec.DecodePos[CurBitLength] + Dist) < Size + ) + { + // Define the code to alphabet number translation. + Dec.QuickNum[Code] = Dec.DecodeNum[Pos]; + } + else + { + // Can be here for length table filled with zeroes only (empty). + Dec.QuickNum[Code] = 0; + } + } + } +} +#endif diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack_async.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack_async.cs new file mode 100644 index 00000000..3edb0ccd --- /dev/null +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack_async.cs @@ -0,0 +1,114 @@ +#if !NETSTANDARD2_0 && !NETFRAMEWORK +using System; +using System.Buffers; +using System.IO; +using System.Threading.Tasks; +using SharpCompress.Common.Rar.Headers; +#if !Rar2017_64bit +using size_t = System.UInt32; +#else +using nint = System.Int64; +using nuint = System.UInt64; +using size_t = System.UInt64; +#endif + +namespace SharpCompress.Compressors.Rar.UnpackV2017; + +internal partial class Unpack : IRarUnpack +{ + private FileHeader fileHeader; + private Stream readStream; + private Stream writeStream; + + private void _UnpackCtor() + { + for (var i = 0; i < AudV.Length; i++) + { + AudV[i] = new AudioVariables(); + } + } + + private int UnpIO_UnpRead(byte[] buf, int offset, int count) => + // NOTE: caller has logic to check for -1 for error we throw instead. + readStream.Read(buf, offset, count); + + private void UnpIO_UnpWrite(byte[] buf, size_t offset, uint count) => + writeStream.Write(buf, checked((int)offset), checked((int)count)); + + + public ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream) + { + // as of 12/2017 .NET limits array indexing to using a signed integer + // MaxWinSize causes unpack to use a fragmented window when the file + // window size exceeds MaxWinSize + // uggh, that's not how this variable is used, it's the size of the currently allocated window buffer + //x MaxWinSize = ((uint)int.MaxValue) + 1; + + // may be long.MaxValue which could indicate unknown size (not present in header) + DestUnpSize = fileHeader.UncompressedSize; + this.fileHeader = fileHeader; + this.readStream = readStream; + this.writeStream = writeStream; + if (!fileHeader.IsStored) + { + Init(fileHeader.WindowSize, fileHeader.IsSolid); + } + Suspended = false; + return DoUnpackAsync(); + } + + public ValueTask DoUnpackAsync() + { + if (fileHeader.IsStored) + { + return UnstoreFileAsync(); + } + else + { + DoUnpack(fileHeader.CompressionAlgorithm, fileHeader.IsSolid); + return new ValueTask(); + } + } + private async ValueTask UnstoreFileAsync() + { + var length = (int)Math.Min(0x10000, DestUnpSize); + var buffer = ArrayPool.Shared.Rent(length); + do + { + var memory = new Memory(buffer, 0, length); + var n = await readStream.ReadAsync(memory); + if (n == 0) + { + break; + } + await writeStream.WriteAsync(memory.Slice(0, n)); + DestUnpSize -= n; + } while (!Suspended); + } + public bool Suspended { get; set; } + + public long DestSize => DestUnpSize; + + public int Char + { + get + { + // TODO: coderb: not sure where the "MAXSIZE-30" comes from, ported from V1 code + if (InAddr > MAX_SIZE - 30) + { + UnpReadBuf(); + } + return InBuf[InAddr++]; + } + } + + public int PpmEscChar + { + get => PPMEscChar; + set => PPMEscChar = value; + } + + public static byte[] EnsureCapacity(byte[] array, int length) => + array.Length < length ? new byte[length] : array; +} +#endif diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index fc6e3d1c..132f18c7 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Readers; @@ -67,7 +68,7 @@ public abstract class AbstractReader : IReader, IReaderExtracti } } - public bool MoveToNextEntry() + public async Task MoveToNextEntryAsync() { if (_completed) { @@ -83,7 +84,7 @@ public abstract class AbstractReader : IReader, IReaderExtracti } if (!_wroteCurrentEntry) { - SkipEntry(); + await SkipEntryAsync(); } _wroteCurrentEntry = false; if (NextEntryForCurrentStream()) @@ -119,15 +120,15 @@ public abstract class AbstractReader : IReader, IReaderExtracti #region Entry Skip/Write - private void SkipEntry() + private async Task SkipEntryAsync() { if (!Entry.IsDirectory) { - Skip(); + await SkipAsync(); } } - private void Skip() + private async Task SkipAsync() { var part = Entry.Parts.First(); @@ -145,11 +146,11 @@ public abstract class AbstractReader : IReader, IReaderExtracti } } //don't know the size so we have to try to decompress to skip - using var s = OpenEntryStream(); - s.SkipEntry(); + using var s = await OpenEntryStreamAsync(); + await s.SkipEntryAsync(); } - public void WriteEntryTo(Stream writableStream) + public async Task WriteEntryToAsync(Stream writableStream) { if (_wroteCurrentEntry) { @@ -167,24 +168,24 @@ public abstract class AbstractReader : IReader, IReaderExtracti ); } - Write(writableStream); + await WriteAsync(writableStream); _wroteCurrentEntry = true; } - internal void Write(Stream writeStream) + internal async Task WriteAsync(Stream writeStream) { var streamListener = this as IReaderExtractionListener; - using Stream s = OpenEntryStream(); + using Stream s = await OpenEntryStreamAsync(); s.TransferTo(writeStream, Entry, streamListener); } - public EntryStream OpenEntryStream() + public async Task OpenEntryStreamAsync() { if (_wroteCurrentEntry) { throw new ArgumentException("WriteEntryTo or OpenEntryStream can only be called once."); } - var stream = GetEntryStream(); + var stream = await GetEntryStreamAsync(); _wroteCurrentEntry = true; return stream; } @@ -192,11 +193,11 @@ public abstract class AbstractReader : IReader, IReaderExtracti /// /// Retains a reference to the entry stream, so we can check whether it completed later. /// - protected EntryStream CreateEntryStream(Stream? decompressed) => - new(this, decompressed.NotNull()); + protected Task CreateEntryStreamAsync(Stream? decompressed) => + Task.FromResult(new EntryStream(this, decompressed.NotNull())); - protected virtual EntryStream GetEntryStream() => - CreateEntryStream(Entry.Parts.First().GetCompressedStream()); + protected virtual Task GetEntryStreamAsync() => + CreateEntryStreamAsync(Entry.Parts.First().GetCompressedStream()); #endregion diff --git a/src/SharpCompress/Readers/IReader.cs b/src/SharpCompress/Readers/IReader.cs index 50fc7f4d..8e880d0f 100644 --- a/src/SharpCompress/Readers/IReader.cs +++ b/src/SharpCompress/Readers/IReader.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Readers; @@ -19,7 +20,7 @@ public interface IReader : IDisposable /// Decompresses the current entry to the stream. This cannot be called twice for the current entry. /// /// - void WriteEntryTo(Stream writableStream); + Task WriteEntryToAsync(Stream writableStream); bool Cancelled { get; } void Cancel(); @@ -28,11 +29,11 @@ public interface IReader : IDisposable /// Moves to the next entry by reading more data from the underlying stream. This skips if data has not been read. /// /// - bool MoveToNextEntry(); + Task MoveToNextEntryAsync(); /// /// Opens the current entry as a stream that will decompress as it is read. /// Read the entire stream or use SkipEntry on EntryStream. /// - EntryStream OpenEntryStream(); + Task OpenEntryStreamAsync(); } diff --git a/src/SharpCompress/Readers/Rar/RarReader.cs b/src/SharpCompress/Readers/Rar/RarReader.cs index aa73e30f..6ac4a912 100644 --- a/src/SharpCompress/Readers/Rar/RarReader.cs +++ b/src/SharpCompress/Readers/Rar/RarReader.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar; using SharpCompress.Compressors.Rar; @@ -77,7 +78,7 @@ public abstract class RarReader : AbstractReader protected virtual IEnumerable CreateFilePartEnumerableForCurrentEntry() => Entry.Parts; - protected override EntryStream GetEntryStream() + protected override async Task GetEntryStreamAsync() { if (Entry.IsRedir) { @@ -90,16 +91,17 @@ public abstract class RarReader : AbstractReader ); if (Entry.IsRarV3) { - return CreateEntryStream(new RarCrcStream(UnpackV1.Value, Entry.FileHeader, stream)); + return await CreateEntryStreamAsync(new RarCrcStream(UnpackV1.Value, Entry.FileHeader, stream)); } if (Entry.FileHeader.FileCrc?.Length > 5) { - return CreateEntryStream( - new RarBLAKE2spStream(UnpackV2017.Value, Entry.FileHeader, stream) + var s = await RarBLAKE2spStream.Create(UnpackV2017.Value, Entry.FileHeader, stream); + return await CreateEntryStreamAsync( + s ); } - return CreateEntryStream(new RarCrcStream(UnpackV2017.Value, Entry.FileHeader, stream)); + return await CreateEntryStreamAsync(new RarCrcStream(UnpackV2017.Value, Entry.FileHeader, stream)); } } diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index 74974850..cc2cb532 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Text; +using System.Threading.Tasks; using SharpCompress.Readers; namespace SharpCompress.Helpers; @@ -173,6 +174,19 @@ internal static class Utility } } + public static async Task SkipAsync(this Stream source) + { + var buffer = GetTransferByteArray(); + try + { + do { } while (await source.ReadAsync(buffer, 0, buffer.Length) == buffer.Length); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + public static bool Find(this Stream source, byte[] array) { var buffer = GetTransferByteArray();