Compare commits

...

13 Commits

Author SHA1 Message Date
copilot-swe-agent[bot]
5a7472146c Add IRarUnpackFactory and refactor RarStream to use factory pattern
- Created IRarUnpackFactory interface with UnpackV1Factory and UnpackV2017Factory implementations
- RarStream now uses factory to create owned IRarUnpack instances for non-solid archives
- RarStream can still accept shared IRarUnpack instance for solid archives (with ownsUnpack flag)
- Updated RarCrcStream and RarBLAKE2spStream to support both factory and shared instance modes
- Solid archives continue to use shared Unpack instance (required for sequential decompression)
- Non-solid archives use factory to create new instances per stream (enables multi-threading)

Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2025-11-29 14:02:28 +00:00
copilot-swe-agent[bot]
3ec4035d55 Fix SupportsMultiThreading to consider solid archives
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2025-11-29 13:26:24 +00:00
copilot-swe-agent[bot]
acffccd889 Refactor RAR to create new Unpack instances per stream for multi-threading support
- For non-solid archives: create new IRarUnpack instance per RarStream to enable multi-threading
- For solid archives: use shared Unpack instance (solid archives require sequential processing)
- Add ownsUnpack parameter to RarStream to properly manage disposal of Unpack instances
- RarStream now disposes Unpack only if it owns the instance

Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2025-11-29 13:21:24 +00:00
copilot-swe-agent[bot]
31fd8f46d8 Fix Windows-only test failure by removing multi-threading support for RAR archives
RAR decompression uses a shared Unpack instance which is not thread-safe.
Removed SupportsMultiThreading override from RarArchive and RarArchiveEntry
classes. Also removed the RAR multi-threaded tests since RAR doesn't support
parallel extraction.

Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2025-11-27 17:31:13 +00:00
copilot-swe-agent[bot]
ac32f2eb1f Initial plan 2025-11-27 16:49:51 +00:00
Adam Hathcock
2e301becb4 fmt 2025-11-27 15:50:25 +00:00
Adam Hathcock
2e7d4eb74b first pass of making Rar multi-threaded 2025-11-27 15:49:29 +00:00
Adam Hathcock
5b2030bb98 add SupportsMultiThreading and IsMultiVolume to archive 2025-11-27 15:07:13 +00:00
Adam Hathcock
c1169539ea add multi-threading test with fix 2025-11-27 12:22:49 +00:00
Adam Hathcock
8d2463f575 More test fixes and fmt 2025-11-27 12:14:35 +00:00
Adam Hathcock
af7e270b2d added SupportsMultiThreading flag for File based Zips 2025-11-27 12:11:10 +00:00
Adam Hathcock
1984da6997 Merge remote-tracking branch 'origin/master' into adam/multi-threaded 2025-11-27 10:55:50 +00:00
Adam Hathcock
4536fddec2 intermediate commit: add zip/filepart that only deals with fileinfo 2025-10-29 13:02:27 +00:00
27 changed files with 461 additions and 165 deletions

View File

@@ -172,4 +172,9 @@ public abstract class AbstractArchive<TEntry, TVolume> : IArchive, IArchiveExtra
return Entries.All(x => x.IsComplete);
}
}
public virtual bool IsMultiVolume =>
_sourceStream?.Files.Count > 1 || _sourceStream?.Streams.Count > 1;
public virtual bool SupportsMultiThreading => false;
}

View File

@@ -45,4 +45,14 @@ public interface IArchive : IDisposable
/// The total size of the files as uncompressed in the archive.
/// </summary>
long TotalUncompressSize { get; }
/// <summary>
/// Is the archive part of a multi-volume set.
/// </summary>
bool IsMultiVolume { get; }
/// <summary>
/// Does the archive support multi-threaded extraction.
/// </summary>
bool SupportsMultiThreading { get; }
}

View File

@@ -88,7 +88,7 @@ public static class IArchiveEntryExtensions
entry,
destinationDirectory,
options,
(x, opt) => entry.WriteToFileAsync(x, opt, cancellationToken),
entry.WriteToFileAsync,
cancellationToken
);
@@ -124,10 +124,10 @@ public static class IArchiveEntryExtensions
entry,
destinationFileName,
options,
async (x, fm) =>
async (x, fm, ct) =>
{
using var fs = File.Open(destinationFileName, fm);
await entry.WriteToAsync(fs, cancellationToken).ConfigureAwait(false);
await entry.WriteToAsync(fs, ct).ConfigureAwait(false);
},
cancellationToken
);

View File

@@ -1,39 +0,0 @@
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using SharpCompress.Common.Rar;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.IO;
using SharpCompress.Readers;
namespace SharpCompress.Archives.Rar;
/// <summary>
/// A rar part based on a FileInfo object
/// </summary>
internal class FileInfoRarArchiveVolume : RarVolume
{
internal FileInfoRarArchiveVolume(FileInfo fileInfo, ReaderOptions options, int index)
: base(StreamingMode.Seekable, fileInfo.OpenRead(), FixOptions(options), index)
{
FileInfo = fileInfo;
FileParts = GetVolumeFileParts().ToArray().ToReadOnly();
}
private static ReaderOptions FixOptions(ReaderOptions options)
{
//make sure we're closing streams with fileinfo
options.LeaveStreamOpen = false;
return options;
}
internal ReadOnlyCollection<RarFilePart> FileParts { get; }
internal FileInfo FileInfo { get; }
internal override RarFilePart CreateFilePart(MarkHeader markHeader, FileHeader fileHeader) =>
new FileInfoRarFilePart(this, ReaderOptions.Password, markHeader, fileHeader, FileInfo);
internal override IEnumerable<RarFilePart> ReadFileParts() => FileParts;
}

View File

@@ -1,21 +0,0 @@
using System.IO;
using SharpCompress.Common.Rar.Headers;
namespace SharpCompress.Archives.Rar;
internal sealed class FileInfoRarFilePart : SeekableFilePart
{
internal FileInfoRarFilePart(
FileInfoRarArchiveVolume volume,
string? password,
MarkHeader mh,
FileHeader fh,
FileInfo fi
)
: base(mh, fh, volume.Index, volume.Stream, password) => FileInfo = fi;
internal FileInfo FileInfo { get; }
internal override string FilePartName =>
"Rar File: " + FileInfo.FullName + " File Entry: " + FileHeader.FileName;
}

View File

@@ -15,6 +15,8 @@ namespace SharpCompress.Archives.Rar;
public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
{
private bool _disposed;
// Shared Unpack instances for solid archives (must be used sequentially)
internal Lazy<IRarUnpack> UnpackV2017 { get; } =
new(() => new Compressors.Rar.UnpackV2017.Unpack());
internal Lazy<IRarUnpack> UnpackV1 { get; } = new(() => new Compressors.Rar.UnpackV1.Unpack());
@@ -47,9 +49,9 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
{
sourceStream.LoadAllParts(); //request all streams
var streams = sourceStream.Streams.ToArray();
var i = 0;
if (streams.Length > 1 && IsRarFile(streams[1], ReaderOptions)) //test part 2 - true = multipart not split
{
var i = 0;
sourceStream.IsVolumes = true;
streams[1].Position = 0;
sourceStream.Position = 0;
@@ -57,12 +59,18 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
return sourceStream.Streams.Select(a => new StreamRarArchiveVolume(
a,
ReaderOptions,
i++
i++,
IsMultiVolume
));
}
//split mode or single file
return new StreamRarArchiveVolume(sourceStream, ReaderOptions, i++).AsEnumerable();
return new StreamRarArchiveVolume(
sourceStream,
ReaderOptions,
0,
IsMultiVolume
).AsEnumerable();
}
protected override IReader CreateReaderForSolidExtraction()
@@ -83,6 +91,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
}
public override bool IsSolid => Volumes.First().IsSolidArchive;
public override bool SupportsMultiThreading => !IsMultiVolume && !IsSolid;
public virtual int MinVersion => Volumes.First().MinVersion;
public virtual int MaxVersion => Volumes.First().MaxVersion;

View File

@@ -70,50 +70,50 @@ public class RarArchiveEntry : RarEntry, IArchiveEntry
public Stream OpenEntryStream()
{
RarStream stream;
if (IsRarV3)
var readStream = new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive);
// For solid archives, use shared Unpack instance (must be processed sequentially)
// For non-solid archives, use factory to create owned instance (supports multi-threading)
if (archive.IsSolid)
{
stream = new RarStream(
archive.UnpackV1.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)
);
var unpack = IsRarV3 ? archive.UnpackV1.Value : archive.UnpackV2017.Value;
var stream = new RarStream(unpack, FileHeader, readStream, ownsUnpack: false);
stream.Initialize();
return stream;
}
else
{
stream = new RarStream(
archive.UnpackV2017.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)
);
var factory = IsRarV3
? (IRarUnpackFactory)UnpackV1Factory.Instance
: UnpackV2017Factory.Instance;
var stream = new RarStream(factory, FileHeader, readStream);
stream.Initialize();
return stream;
}
stream.Initialize();
return stream;
}
public async Task<Stream> OpenEntryStreamAsync(CancellationToken cancellationToken = default)
{
RarStream stream;
if (IsRarV3)
var readStream = new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive);
// For solid archives, use shared Unpack instance (must be processed sequentially)
// For non-solid archives, use factory to create owned instance (supports multi-threading)
if (archive.IsSolid)
{
stream = new RarStream(
archive.UnpackV1.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)
);
var unpack = IsRarV3 ? archive.UnpackV1.Value : archive.UnpackV2017.Value;
var stream = new RarStream(unpack, FileHeader, readStream, ownsUnpack: false);
await stream.InitializeAsync(cancellationToken);
return stream;
}
else
{
stream = new RarStream(
archive.UnpackV2017.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)
);
var factory = IsRarV3
? (IRarUnpackFactory)UnpackV1Factory.Instance
: UnpackV2017Factory.Instance;
var stream = new RarStream(factory, FileHeader, readStream);
await stream.InitializeAsync(cancellationToken);
return stream;
}
await stream.InitializeAsync(cancellationToken);
return stream;
}
public bool IsComplete
@@ -134,4 +134,7 @@ public class RarArchiveEntry : RarEntry, IArchiveEntry
);
}
}
public override bool SupportsMultiThreading =>
!archive.IsSolid && Parts.Single().SupportsMultiThreading;
}

View File

@@ -1,25 +1,29 @@
using System.IO;
using SharpCompress.Common.Rar;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.IO;
namespace SharpCompress.Archives.Rar;
internal class SeekableFilePart : RarFilePart
internal class SeekableRarFilePart : RarFilePart
{
private readonly Stream _stream;
private readonly string? _password;
private readonly bool _isMultiVolume;
internal SeekableFilePart(
internal SeekableRarFilePart(
MarkHeader mh,
FileHeader fh,
int index,
Stream stream,
string? password
string? password,
bool isMultiVolume
)
: base(mh, fh, index)
{
_stream = stream;
_password = password;
_isMultiVolume = isMultiVolume;
}
internal override Stream GetCompressedStream()
@@ -42,4 +46,7 @@ internal class SeekableFilePart : RarFilePart
}
internal override string FilePartName => "Unknown Stream - File Entry: " + FileHeader.FileName;
public override bool SupportsMultiThreading =>
!_isMultiVolume && _stream is SourceStream ss && ss.IsFileMode && ss.Files.Count == 1;
}

View File

@@ -9,11 +9,28 @@ namespace SharpCompress.Archives.Rar;
internal class StreamRarArchiveVolume : RarVolume
{
internal StreamRarArchiveVolume(Stream stream, ReaderOptions options, int index)
: base(StreamingMode.Seekable, stream, options, index) { }
private readonly bool _isMultiVolume;
internal StreamRarArchiveVolume(
Stream stream,
ReaderOptions options,
int index,
bool isMultiVolume
)
: base(StreamingMode.Seekable, stream, options, index)
{
_isMultiVolume = isMultiVolume;
}
internal override IEnumerable<RarFilePart> ReadFileParts() => GetVolumeFileParts();
internal override RarFilePart CreateFilePart(MarkHeader markHeader, FileHeader fileHeader) =>
new SeekableFilePart(markHeader, fileHeader, Index, Stream, ReaderOptions.Password);
new SeekableRarFilePart(
markHeader,
fileHeader,
Index,
Stream,
ReaderOptions.Password,
_isMultiVolume
);
}

View File

@@ -283,7 +283,12 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
yield return new ZipArchiveEntry(
this,
new SeekableZipFilePart(headerFactory.NotNull(), deh, s)
new SeekableZipFilePart(
headerFactory.NotNull(),
deh,
s,
IsMultiVolume
)
);
}
break;
@@ -385,4 +390,6 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
((IStreamStack)stream).StackSeek(0);
return ZipReader.Open(stream, ReaderOptions, Entries);
}
public override bool SupportsMultiThreading => !IsMultiVolume;
}

View File

@@ -23,5 +23,7 @@ public class ZipArchiveEntry : ZipEntry, IArchiveEntry
public bool IsComplete => true;
public override bool SupportsMultiThreading => Parts.Single().SupportsMultiThreading;
#endregion
}

View File

@@ -87,4 +87,5 @@ public abstract class Entry : IEntry
/// Entry file attribute.
/// </summary>
public virtual int? Attrib => throw new NotImplementedException();
public virtual bool SupportsMultiThreading => false;
}

View File

@@ -128,7 +128,7 @@ internal static class ExtractionMethods
IEntry entry,
string destinationDirectory,
ExtractionOptions? options,
Func<string, ExtractionOptions?, Task> writeAsync,
Func<string, ExtractionOptions?, CancellationToken, Task> writeAsync,
CancellationToken cancellationToken = default
)
{
@@ -189,7 +189,7 @@ internal static class ExtractionMethods
"Entry is trying to write a file outside of the destination directory."
);
}
await writeAsync(destinationFileName, options).ConfigureAwait(false);
await writeAsync(destinationFileName, options, cancellationToken).ConfigureAwait(false);
}
else if (options.ExtractFullPath && !Directory.Exists(destinationFileName))
{
@@ -201,7 +201,7 @@ internal static class ExtractionMethods
IEntry entry,
string destinationFileName,
ExtractionOptions? options,
Func<string, FileMode, Task> openAndWriteAsync,
Func<string, FileMode, CancellationToken, Task> openAndWriteAsync,
CancellationToken cancellationToken = default
)
{
@@ -225,7 +225,8 @@ internal static class ExtractionMethods
fm = FileMode.CreateNew;
}
await openAndWriteAsync(destinationFileName, fm).ConfigureAwait(false);
await openAndWriteAsync(destinationFileName, fm, cancellationToken)
.ConfigureAwait(false);
entry.PreserveExtractionOptions(destinationFileName, options);
}
}

View File

@@ -14,4 +14,6 @@ public abstract class FilePart
internal abstract Stream? GetCompressedStream();
internal abstract Stream? GetRawStream();
internal bool Skipped { get; set; }
public virtual bool SupportsMultiThreading => false;
}

View File

@@ -21,4 +21,5 @@ public interface IEntry
DateTime? LastModifiedTime { get; }
long Size { get; }
int? Attrib { get; }
bool SupportsMultiThreading { get; }
}

View File

@@ -1,5 +1,6 @@
using System.IO;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
namespace SharpCompress.Common.Zip;
@@ -7,13 +8,19 @@ internal class SeekableZipFilePart : ZipFilePart
{
private bool _isLocalHeaderLoaded;
private readonly SeekableZipHeaderFactory _headerFactory;
private readonly bool _isMultiVolume;
internal SeekableZipFilePart(
SeekableZipHeaderFactory headerFactory,
DirectoryEntryHeader header,
Stream stream
Stream stream,
bool isMultiVolume
)
: base(header, stream) => _headerFactory = headerFactory;
: base(header, stream)
{
_headerFactory = headerFactory;
_isMultiVolume = isMultiVolume;
}
internal override Stream GetCompressedStream()
{
@@ -30,8 +37,20 @@ internal class SeekableZipFilePart : ZipFilePart
protected override Stream CreateBaseStream()
{
if (!_isMultiVolume && BaseStream is SourceStream ss)
{
if (ss.IsFileMode && ss.Files.Count == 1)
{
var fileStream = ss.CurrentFile.OpenRead();
fileStream.Position = Header.DataStartPosition.NotNull();
return fileStream;
}
}
BaseStream.Position = Header.DataStartPosition.NotNull();
return BaseStream;
}
public override bool SupportsMultiThreading =>
!_isMultiVolume && BaseStream is SourceStream ss && ss.IsFileMode && ss.Files.Count == 1;
}

View File

@@ -0,0 +1,34 @@
namespace SharpCompress.Compressors.Rar;
/// <summary>
/// Factory interface for creating IRarUnpack instances.
/// Each created instance is owned by the caller and should be disposed when done.
/// </summary>
internal interface IRarUnpackFactory
{
/// <summary>
/// Creates a new IRarUnpack instance.
/// The caller is responsible for disposing the returned instance.
/// </summary>
IRarUnpack Create();
}
/// <summary>
/// Factory for creating UnpackV1 instances (RAR v3 and earlier).
/// </summary>
internal sealed class UnpackV1Factory : IRarUnpackFactory
{
public static readonly UnpackV1Factory Instance = new();
public IRarUnpack Create() => new UnpackV1.Unpack();
}
/// <summary>
/// Factory for creating UnpackV2017 instances (RAR v5+).
/// </summary>
internal sealed class UnpackV2017Factory : IRarUnpackFactory
{
public static readonly UnpackV2017Factory Instance = new();
public IRarUnpack Create() => new UnpackV2017.Unpack();
}

View File

@@ -62,10 +62,6 @@ internal sealed class MultiVolumeReadOnlyStream : Stream, IStreamStack
base.Dispose(disposing);
if (disposing)
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(MultiVolumeReadOnlyStream));
#endif
if (filePartEnumerator != null)
{
filePartEnumerator.Dispose();

View File

@@ -106,11 +106,30 @@ internal class RarBLAKE2spStream : RarStream, IStreamStack
byte[] _hash = { };
private RarBLAKE2spStream(
IRarUnpack unpack,
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream
)
: base(unpack, fileHeader, readStream)
: base(unpackFactory, fileHeader, readStream)
{
this.readStream = readStream;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RarBLAKE2spStream));
#endif
disableCRCCheck = fileHeader.IsEncrypted;
_hash = fileHeader.FileCrc.NotNull();
_blake2sp = new BLAKE2SP();
ResetCrc();
}
private RarBLAKE2spStream(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack
)
: base(unpack, fileHeader, readStream, ownsUnpack)
{
this.readStream = readStream;
@@ -124,24 +143,49 @@ internal class RarBLAKE2spStream : RarStream, IStreamStack
}
public static RarBLAKE2spStream Create(
IRarUnpack unpack,
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream
)
{
var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream);
var stream = new RarBLAKE2spStream(unpackFactory, fileHeader, readStream);
stream.Initialize();
return stream;
}
public static RarBLAKE2spStream Create(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack
)
{
var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream, ownsUnpack);
stream.Initialize();
return stream;
}
public static async Task<RarBLAKE2spStream> CreateAsync(
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
CancellationToken cancellationToken = default
)
{
var stream = new RarBLAKE2spStream(unpackFactory, fileHeader, readStream);
await stream.InitializeAsync(cancellationToken);
return stream;
}
public static async Task<RarBLAKE2spStream> CreateAsync(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack,
CancellationToken cancellationToken = default
)
{
var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream);
var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream, ownsUnpack);
await stream.InitializeAsync(cancellationToken);
return stream;
}

View File

@@ -34,11 +34,27 @@ internal class RarCrcStream : RarStream, IStreamStack
private readonly bool disableCRC;
private RarCrcStream(
IRarUnpack unpack,
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream
)
: base(unpack, fileHeader, readStream)
: base(unpackFactory, fileHeader, readStream)
{
this.readStream = readStream;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RarCrcStream));
#endif
disableCRC = fileHeader.IsEncrypted;
ResetCrc();
}
private RarCrcStream(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack
)
: base(unpack, fileHeader, readStream, ownsUnpack)
{
this.readStream = readStream;
#if DEBUG_STREAMS
@@ -49,24 +65,49 @@ internal class RarCrcStream : RarStream, IStreamStack
}
public static RarCrcStream Create(
IRarUnpack unpack,
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream
)
{
var stream = new RarCrcStream(unpack, fileHeader, readStream);
var stream = new RarCrcStream(unpackFactory, fileHeader, readStream);
stream.Initialize();
return stream;
}
public static RarCrcStream Create(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack
)
{
var stream = new RarCrcStream(unpack, fileHeader, readStream, ownsUnpack);
stream.Initialize();
return stream;
}
public static async Task<RarCrcStream> CreateAsync(
IRarUnpackFactory unpackFactory,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
CancellationToken cancellationToken = default
)
{
var stream = new RarCrcStream(unpackFactory, fileHeader, readStream);
await stream.InitializeAsync(cancellationToken);
return stream;
}
public static async Task<RarCrcStream> CreateAsync(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream,
bool ownsUnpack,
CancellationToken cancellationToken = default
)
{
var stream = new RarCrcStream(unpack, fileHeader, readStream);
var stream = new RarCrcStream(unpack, fileHeader, readStream, ownsUnpack);
await stream.InitializeAsync(cancellationToken);
return stream;
}

View File

@@ -35,6 +35,7 @@ internal class RarStream : Stream, IStreamStack
private readonly IRarUnpack unpack;
private readonly FileHeader fileHeader;
private readonly Stream readStream;
private readonly bool ownsUnpack;
private bool fetch;
@@ -49,11 +50,28 @@ internal class RarStream : Stream, IStreamStack
private bool isDisposed;
private long _position;
public RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream)
/// <summary>
/// Creates a new RarStream that owns and will dispose its IRarUnpack instance.
/// </summary>
/// <param name="unpackFactory">Factory to create the IRarUnpack instance</param>
/// <param name="fileHeader">File header for the entry</param>
/// <param name="readStream">Stream to read compressed data from</param>
public RarStream(IRarUnpackFactory unpackFactory, FileHeader fileHeader, Stream readStream)
: this(unpackFactory.Create(), fileHeader, readStream, ownsUnpack: true) { }
/// <summary>
/// Creates a new RarStream with the specified unpack instance.
/// </summary>
/// <param name="unpack">The IRarUnpack instance to use</param>
/// <param name="fileHeader">File header for the entry</param>
/// <param name="readStream">Stream to read compressed data from</param>
/// <param name="ownsUnpack">Whether this stream should dispose the unpack instance</param>
internal RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream, bool ownsUnpack)
{
this.unpack = unpack;
this.fileHeader = fileHeader;
this.readStream = readStream;
this.ownsUnpack = ownsUnpack;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RarStream));
@@ -82,11 +100,14 @@ internal class RarStream : Stream, IStreamStack
{
if (disposing)
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(RarStream));
#endif
ArrayPool<byte>.Shared.Return(this.tmpBuffer);
this.tmpBuffer = null;
// Dispose the unpack instance if we own it
if (ownsUnpack && unpack is IDisposable disposableUnpack)
{
disposableUnpack.Dispose();
}
}
isDisposed = true;
base.Dispose(disposing);

View File

@@ -15,7 +15,7 @@ public class SourceStream : Stream, IStreamStack
#endif
int IStreamStack.DefaultBufferSize { get; set; }
Stream IStreamStack.BaseStream() => _streams[_stream];
Stream IStreamStack.BaseStream() => _streams[_streamIndex];
int IStreamStack.BufferSize
{
@@ -35,7 +35,7 @@ public class SourceStream : Stream, IStreamStack
private readonly List<Stream> _streams;
private readonly Func<int, FileInfo?>? _getFilePart;
private readonly Func<int, Stream?>? _getStreamPart;
private int _stream;
private int _streamIndex;
public SourceStream(FileInfo file, Func<int, FileInfo?> getPart, ReaderOptions options)
: this(null, null, file, getPart, options) { }
@@ -59,7 +59,7 @@ public class SourceStream : Stream, IStreamStack
if (!IsFileMode)
{
_streams.Add(stream!);
_streams.Add(stream.NotNull("stream is null"));
_getStreamPart = getStreamPart;
_getFilePart = _ => null;
if (stream is FileStream fileStream)
@@ -69,12 +69,12 @@ public class SourceStream : Stream, IStreamStack
}
else
{
_files.Add(file!);
_files.Add(file.NotNull("file is null"));
_streams.Add(_files[0].OpenRead());
_getFilePart = getFilePart;
_getStreamPart = _ => null;
}
_stream = 0;
_streamIndex = 0;
_prevSize = 0;
#if DEBUG_STREAMS
@@ -93,10 +93,12 @@ public class SourceStream : Stream, IStreamStack
public ReaderOptions ReaderOptions { get; }
public bool IsFileMode { get; }
public IEnumerable<FileInfo> Files => _files;
public IEnumerable<Stream> Streams => _streams;
public IReadOnlyList<FileInfo> Files => _files;
public IReadOnlyList<Stream> Streams => _streams;
private Stream Current => _streams[_stream];
private Stream Current => _streams[_streamIndex];
public FileInfo CurrentFile => _files[_streamIndex];
public bool LoadStream(int index) //ensure all parts to id are loaded
{
@@ -107,7 +109,7 @@ public class SourceStream : Stream, IStreamStack
var f = _getFilePart.NotNull("GetFilePart is null")(_streams.Count);
if (f == null)
{
_stream = _streams.Count - 1;
_streamIndex = _streams.Count - 1;
return false;
}
//throw new Exception($"File part {idx} not available.");
@@ -119,7 +121,7 @@ public class SourceStream : Stream, IStreamStack
var s = _getStreamPart.NotNull("GetStreamPart is null")(_streams.Count);
if (s == null)
{
_stream = _streams.Count - 1;
_streamIndex = _streams.Count - 1;
return false;
}
//throw new Exception($"Stream part {idx} not available.");
@@ -137,10 +139,10 @@ public class SourceStream : Stream, IStreamStack
{
if (LoadStream(idx))
{
_stream = idx;
_streamIndex = idx;
}
return _stream == idx;
return _streamIndex == idx;
}
public override bool CanRead => true;
@@ -184,7 +186,7 @@ public class SourceStream : Stream, IStreamStack
var length = Current.Length;
// Load next file if present
if (!SetStream(_stream + 1))
if (!SetStream(_streamIndex + 1))
{
break;
}
@@ -223,7 +225,7 @@ public class SourceStream : Stream, IStreamStack
while (_prevSize + Current.Length < pos)
{
_prevSize += Current.Length;
SetStream(_stream + 1);
SetStream(_streamIndex + 1);
}
}
@@ -273,7 +275,7 @@ public class SourceStream : Stream, IStreamStack
var length = Current.Length;
// Load next file if present
if (!SetStream(_stream + 1))
if (!SetStream(_streamIndex + 1))
{
break;
}
@@ -322,7 +324,7 @@ public class SourceStream : Stream, IStreamStack
var length = Current.Length;
// Load next file if present
if (!SetStream(_stream + 1))
if (!SetStream(_streamIndex + 1))
{
break;
}

View File

@@ -82,7 +82,7 @@ public static class IReaderExtensions
reader.Entry,
destinationDirectory,
options,
(fileName, opts) => reader.WriteEntryToFileAsync(fileName, opts, cancellationToken),
reader.WriteEntryToFileAsync,
cancellationToken
)
.ConfigureAwait(false);
@@ -101,10 +101,10 @@ public static class IReaderExtensions
reader.Entry,
destinationFileName,
options,
async (x, fm) =>
async (x, fm, ct) =>
{
using var fs = File.Open(destinationFileName, fm);
await reader.WriteEntryToAsync(fs, cancellationToken).ConfigureAwait(false);
await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false);
},
cancellationToken
)

View File

@@ -15,6 +15,8 @@ public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
{
private bool _disposed;
private RarVolume? volume;
// Shared Unpack instances for solid archives (must be used sequentially)
private Lazy<IRarUnpack> UnpackV2017 { get; } =
new(() => new Compressors.Rar.UnpackV2017.Unpack());
private Lazy<IRarUnpack> UnpackV1 { get; } = new(() => new Compressors.Rar.UnpackV1.Unpack());
@@ -111,19 +113,50 @@ public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
CreateFilePartEnumerableForCurrentEntry().Cast<RarFilePart>(),
this
);
if (Entry.IsRarV3)
{
return CreateEntryStream(RarCrcStream.Create(UnpackV1.Value, Entry.FileHeader, stream));
}
if (Entry.FileHeader.FileCrc?.Length > 5)
// For solid archives, use shared Unpack instance (must be processed sequentially)
// For non-solid archives, use factory to create owned instance
if (Entry.IsSolid || Entry.FileHeader.IsSolid)
{
var unpack = Entry.IsRarV3 ? UnpackV1.Value : UnpackV2017.Value;
if (Entry.IsRarV3)
{
return CreateEntryStream(
RarCrcStream.Create(unpack, Entry.FileHeader, stream, ownsUnpack: false)
);
}
if (Entry.FileHeader.FileCrc?.Length > 5)
{
return CreateEntryStream(
RarBLAKE2spStream.Create(unpack, Entry.FileHeader, stream, ownsUnpack: false)
);
}
return CreateEntryStream(
RarBLAKE2spStream.Create(UnpackV2017.Value, Entry.FileHeader, stream)
RarCrcStream.Create(unpack, Entry.FileHeader, stream, ownsUnpack: false)
);
}
else
{
var factory = Entry.IsRarV3
? (IRarUnpackFactory)UnpackV1Factory.Instance
: UnpackV2017Factory.Instance;
return CreateEntryStream(RarCrcStream.Create(UnpackV2017.Value, Entry.FileHeader, stream));
if (Entry.IsRarV3)
{
return CreateEntryStream(RarCrcStream.Create(factory, Entry.FileHeader, stream));
}
if (Entry.FileHeader.FileCrc?.Length > 5)
{
return CreateEntryStream(
RarBLAKE2spStream.Create(factory, Entry.FileHeader, stream)
);
}
return CreateEntryStream(RarCrcStream.Create(factory, Entry.FileHeader, stream));
}
}
protected override async System.Threading.Tasks.Task<EntryStream> GetEntryStreamAsync(
@@ -139,28 +172,83 @@ public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
CreateFilePartEnumerableForCurrentEntry().Cast<RarFilePart>(),
this
);
if (Entry.IsRarV3)
// For solid archives, use shared Unpack instance (must be processed sequentially)
// For non-solid archives, use factory to create owned instance
if (Entry.IsSolid || Entry.FileHeader.IsSolid)
{
var unpack = Entry.IsRarV3 ? UnpackV1.Value : UnpackV2017.Value;
if (Entry.IsRarV3)
{
return CreateEntryStream(
await RarCrcStream
.CreateAsync(
unpack,
Entry.FileHeader,
stream,
ownsUnpack: false,
cancellationToken
)
.ConfigureAwait(false)
);
}
if (Entry.FileHeader.FileCrc?.Length > 5)
{
return CreateEntryStream(
await RarBLAKE2spStream
.CreateAsync(
unpack,
Entry.FileHeader,
stream,
ownsUnpack: false,
cancellationToken
)
.ConfigureAwait(false)
);
}
return CreateEntryStream(
await RarCrcStream
.CreateAsync(UnpackV1.Value, Entry.FileHeader, stream, cancellationToken)
.CreateAsync(
unpack,
Entry.FileHeader,
stream,
ownsUnpack: false,
cancellationToken
)
.ConfigureAwait(false)
);
}
if (Entry.FileHeader.FileCrc?.Length > 5)
else
{
var factory = Entry.IsRarV3
? (IRarUnpackFactory)UnpackV1Factory.Instance
: UnpackV2017Factory.Instance;
if (Entry.IsRarV3)
{
return CreateEntryStream(
await RarCrcStream
.CreateAsync(factory, Entry.FileHeader, stream, cancellationToken)
.ConfigureAwait(false)
);
}
if (Entry.FileHeader.FileCrc?.Length > 5)
{
return CreateEntryStream(
await RarBLAKE2spStream
.CreateAsync(factory, Entry.FileHeader, stream, cancellationToken)
.ConfigureAwait(false)
);
}
return CreateEntryStream(
await RarBLAKE2spStream
.CreateAsync(UnpackV2017.Value, Entry.FileHeader, stream, cancellationToken)
await RarCrcStream
.CreateAsync(factory, Entry.FileHeader, stream, cancellationToken)
.ConfigureAwait(false)
);
}
return CreateEntryStream(
await RarCrcStream
.CreateAsync(UnpackV2017.Value, Entry.FileHeader, stream, cancellationToken)
.ConfigureAwait(false)
);
}
}

View File

@@ -134,6 +134,7 @@ public class ArchiveTests : ReaderTests
{
foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
{
Assert.False(entry.SupportsMultiThreading);
entry.WriteToDirectory(
SCRATCH_FILES_PATH,
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
@@ -266,6 +267,31 @@ public class ArchiveTests : ReaderTests
VerifyFiles();
}
protected async Task ArchiveFileRead_Multithreaded(
IArchiveFactory archiveFactory,
string testArchive,
ReaderOptions? readerOptions = null
)
{
testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive);
var tasks = new List<Task>();
using (var archive = archiveFactory.Open(new FileInfo(testArchive), readerOptions))
{
Assert.True(archive.SupportsMultiThreading);
foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
{
Assert.True(entry.SupportsMultiThreading);
var t = entry.WriteToDirectoryAsync(
SCRATCH_FILES_PATH,
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
);
tasks.Add(t);
}
}
await Task.WhenAll(tasks);
VerifyFiles();
}
protected void ArchiveFileRead(
IArchiveFactory archiveFactory,
string testArchive,
@@ -289,6 +315,11 @@ public class ArchiveTests : ReaderTests
protected void ArchiveFileRead(string testArchive, ReaderOptions? readerOptions = null) =>
ArchiveFileRead(ArchiveFactory.AutoFactory, testArchive, readerOptions);
protected Task ArchiveFileRead_Multithreaded(
string testArchive,
ReaderOptions? readerOptions = null
) => ArchiveFileRead_Multithreaded(ArchiveFactory.AutoFactory, testArchive, readerOptions);
protected void ArchiveFileSkip(
string testArchive,
string fileOrder,

View File

@@ -1,5 +1,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Archives.Rar;
using SharpCompress.Common;
@@ -292,9 +293,15 @@ public class RarArchiveTests : ArchiveTests
[Fact]
public void Rar_ArchiveFileRead() => ArchiveFileRead("Rar.rar");
[Fact]
public Task Rar_ArchiveFileRead_Multithreaded() => ArchiveFileRead_Multithreaded("Rar.rar");
[Fact]
public void Rar5_ArchiveFileRead() => ArchiveFileRead("Rar5.rar");
[Fact]
public Task Rar5_ArchiveFileRead_Multithreaded() => ArchiveFileRead_Multithreaded("Rar5.rar");
[Fact]
public void Rar_ArchiveFileRead_HasDirectories() =>
DoRar_ArchiveFileRead_HasDirectories("Rar.rar");
@@ -359,6 +366,9 @@ public class RarArchiveTests : ArchiveTests
[Fact]
public void Rar2_ArchiveFileRead() => ArchiveFileRead("Rar2.rar");
[Fact]
public Task Rar2_ArchiveFileRead_Multithreaded() => ArchiveFileRead_Multithreaded("Rar2.rar");
[Fact]
public void Rar15_ArchiveFileRead()
{

View File

@@ -2,6 +2,7 @@ using System;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
@@ -88,6 +89,10 @@ public class ZipArchiveTests : ArchiveTests
[Fact]
public void Zip_Deflate_ArchiveFileRead() => ArchiveFileRead("Zip.deflate.zip");
[Fact]
public Task Zip_Deflate_ArchiveFileRead_Multithreaded() =>
ArchiveFileRead_Multithreaded("Zip.deflate.zip");
[Fact]
public void Zip_Deflate_ArchiveExtractToDirectory() =>
ArchiveExtractToDirectory("Zip.deflate.zip");