diff --git a/build/Program.cs b/build/Program.cs index 47a86745..1ad21f50 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -230,7 +230,7 @@ static async Task<(string version, bool isPrerelease)> GetVersion() } else { - // Not tagged - create prerelease version based on next minor version + // Not tagged - create prerelease version var allTags = (await GetGitOutput("tag", "--list")) .Split('\n', StringSplitOptions.RemoveEmptyEntries) .Where(tag => Regex.IsMatch(tag.Trim(), @"^\d+\.\d+\.\d+$")) @@ -240,8 +240,22 @@ static async Task<(string version, bool isPrerelease)> GetVersion() var lastTag = allTags.OrderBy(tag => Version.Parse(tag)).LastOrDefault() ?? "0.0.0"; var lastVersion = Version.Parse(lastTag); - // Increment minor version for next release - var nextVersion = new Version(lastVersion.Major, lastVersion.Minor + 1, 0); + // Determine version increment based on branch + var currentBranch = await GetCurrentBranch(); + Version nextVersion; + + if (currentBranch == "release") + { + // Release branch: increment patch version + nextVersion = new Version(lastVersion.Major, lastVersion.Minor, lastVersion.Build + 1); + Console.WriteLine($"Building prerelease for release branch (patch increment)"); + } + else + { + // Master or other branches: increment minor version + nextVersion = new Version(lastVersion.Major, lastVersion.Minor + 1, 0); + Console.WriteLine($"Building prerelease for {currentBranch} branch (minor increment)"); + } // Use commit count since the last version tag if available; otherwise, fall back to total count var revListArgs = allTags.Any() ? $"--count {lastTag}..HEAD" : "--count HEAD"; @@ -253,6 +267,28 @@ static async Task<(string version, bool isPrerelease)> GetVersion() } } +static async Task GetCurrentBranch() +{ + // In GitHub Actions, GITHUB_REF_NAME contains the branch name + var githubRefName = Environment.GetEnvironmentVariable("GITHUB_REF_NAME"); + if (!string.IsNullOrEmpty(githubRefName)) + { + return githubRefName; + } + + // Fallback to git command for local builds + try + { + var (output, _) = await ReadAsync("git", "branch --show-current"); + return output.Trim(); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Could not determine current branch: {ex.Message}"); + return "unknown"; + } +} + static async Task GetGitOutput(string command, string args) { try diff --git a/src/SharpCompress/Archives/ArchiveFactory.cs b/src/SharpCompress/Archives/ArchiveFactory.cs index eca2c579..3c69445a 100644 --- a/src/SharpCompress/Archives/ArchiveFactory.cs +++ b/src/SharpCompress/Archives/ArchiveFactory.cs @@ -150,24 +150,14 @@ public static partial class ArchiveFactory ); } - // Async methods moved to ArchiveFactory.Async.cs - - public static bool IsArchive( - string filePath, - out ArchiveType? type, - int bufferSize = ReaderOptions.DefaultBufferSize - ) + public static bool IsArchive(string filePath, out ArchiveType? type) { filePath.NotNullOrEmpty(nameof(filePath)); using Stream s = File.OpenRead(filePath); - return IsArchive(s, out type, bufferSize); + return IsArchive(s, out type); } - public static bool IsArchive( - Stream stream, - out ArchiveType? type, - int bufferSize = ReaderOptions.DefaultBufferSize - ) + public static bool IsArchive(Stream stream, out ArchiveType? type) { type = null; stream.NotNull(nameof(stream)); diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 3bf94035..d61d03da 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -9,8 +9,6 @@ namespace SharpCompress.Archives; public static class IArchiveEntryExtensions { - private const int BufferSize = 81920; - /// The archive entry to extract. extension(IArchiveEntry archiveEntry) { @@ -28,7 +26,7 @@ public static class IArchiveEntryExtensions using var entryStream = archiveEntry.OpenEntryStream(); var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); - sourceStream.CopyTo(streamToWriteTo, BufferSize); + sourceStream.CopyTo(streamToWriteTo, Constants.BufferSize); } /// @@ -51,7 +49,7 @@ public static class IArchiveEntryExtensions using var entryStream = await archiveEntry.OpenEntryStreamAsync(cancellationToken); var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress); await sourceStream - .CopyToAsync(streamToWriteTo, BufferSize, cancellationToken) + .CopyToAsync(streamToWriteTo, Constants.BufferSize, cancellationToken) .ConfigureAwait(false); } } diff --git a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs index 799644d8..1ea0b3d3 100644 --- a/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs +++ b/src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs @@ -12,65 +12,157 @@ using SharpCompress.Readers; namespace SharpCompress.Archives.SevenZip; -public partial class SevenZipArchive : AbstractArchive +public class SevenZipArchive : AbstractArchive { private ArchiveDatabase? _database; + /// + /// Constructor expects a filepath to an existing file. + /// + /// + /// + public static SevenZipArchive Open(string filePath, ReaderOptions? readerOptions = null) + { + filePath.NotNullOrEmpty("filePath"); + return Open(new FileInfo(filePath), readerOptions ?? new ReaderOptions()); + } + + /// + /// Constructor with a FileInfo object to an existing file. + /// + /// + /// + public static SevenZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null) + { + fileInfo.NotNull("fileInfo"); + return new SevenZipArchive( + new SourceStream( + fileInfo, + i => ArchiveVolumeFactory.GetFilePart(i, fileInfo), + readerOptions ?? new ReaderOptions() + ) + ); + } + + /// + /// Constructor with all file parts passed in + /// + /// + /// + public static SevenZipArchive Open( + IEnumerable fileInfos, + ReaderOptions? readerOptions = null + ) + { + fileInfos.NotNull(nameof(fileInfos)); + var files = fileInfos.ToArray(); + return new SevenZipArchive( + new SourceStream( + files[0], + i => i < files.Length ? files[i] : null, + readerOptions ?? new ReaderOptions() + ) + ); + } + + /// + /// Constructor with all stream parts passed in + /// + /// + /// + public static SevenZipArchive Open( + IEnumerable streams, + ReaderOptions? readerOptions = null + ) + { + streams.NotNull(nameof(streams)); + var strms = streams.ToArray(); + return new SevenZipArchive( + new SourceStream( + strms[0], + i => i < strms.Length ? strms[i] : null, + readerOptions ?? new ReaderOptions() + ) + ); + } + + /// + /// Takes a seekable Stream as a source + /// + /// + /// + public static SevenZipArchive Open(Stream stream, ReaderOptions? readerOptions = null) + { + stream.NotNull("stream"); + + if (stream is not { CanSeek: true }) + { + throw new ArgumentException("Stream must be seekable", nameof(stream)); + } + + return new SevenZipArchive( + new SourceStream(stream, _ => null, readerOptions ?? new ReaderOptions()) + ); + } + + /// + /// Constructor with a SourceStream able to handle FileInfo and Streams. + /// + /// private SevenZipArchive(SourceStream sourceStream) : base(ArchiveType.SevenZip, sourceStream) { } - internal SevenZipArchive() - : base(ArchiveType.SevenZip) { } - protected override IEnumerable LoadVolumes(SourceStream sourceStream) { - sourceStream.NotNull("SourceStream is null").LoadAllParts(); - return new SevenZipVolume(sourceStream, ReaderOptions, 0).AsEnumerable(); + sourceStream.NotNull("SourceStream is null").LoadAllParts(); //request all streams + return new SevenZipVolume(sourceStream, ReaderOptions, 0).AsEnumerable(); //simple single volume or split, multivolume not supported } + public static bool IsSevenZipFile(string filePath) => IsSevenZipFile(new FileInfo(filePath)); + + public static bool IsSevenZipFile(FileInfo fileInfo) + { + if (!fileInfo.Exists) + { + return false; + } + using Stream stream = fileInfo.OpenRead(); + return IsSevenZipFile(stream); + } + + internal SevenZipArchive() + : base(ArchiveType.SevenZip) { } + protected override IEnumerable LoadEntries( IEnumerable volumes ) { - foreach (var volume in volumes) + var stream = volumes.Single().Stream; + LoadFactory(stream); + if (_database is null) { - LoadFactory(volume.Stream); - if (_database is null) + return Enumerable.Empty(); + } + var entries = new SevenZipArchiveEntry[_database._files.Count]; + for (var i = 0; i < _database._files.Count; i++) + { + var file = _database._files[i]; + entries[i] = new SevenZipArchiveEntry( + this, + new SevenZipFilePart(stream, _database, i, file, ReaderOptions.ArchiveEncoding) + ); + } + foreach (var group in entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Folder)) + { + var isSolid = false; + foreach (var entry in group) { - yield break; - } - var entries = new SevenZipArchiveEntry[_database._files.Count]; - for (var i = 0; i < _database._files.Count; i++) - { - var file = _database._files[i]; - entries[i] = new SevenZipArchiveEntry( - this, - new SevenZipFilePart( - volume.Stream, - _database, - i, - file, - ReaderOptions.ArchiveEncoding - ) - ); - } - foreach ( - var group in entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Folder) - ) - { - var isSolid = false; - foreach (var entry in group) - { - entry.IsSolid = isSolid; - isSolid = true; - } - } - - foreach (var entry in entries) - { - yield return entry; + entry.IsSolid = isSolid; + isSolid = true; //mark others in this group as solid - same as rar behaviour. } } + + return entries; } private void LoadFactory(Stream stream) @@ -84,6 +176,28 @@ public partial class SevenZipArchive : AbstractArchive Signature => + new byte[] { (byte)'7', (byte)'z', 0xBC, 0xAF, 0x27, 0x1C }; + + private static bool SignatureMatch(Stream stream) + { + var reader = new BinaryReader(stream); + ReadOnlySpan signatureBytes = reader.ReadBytes(6); + return signatureBytes.SequenceEqual(Signature); + } + protected override IReader CreateReaderForSolidExtraction() => new SevenZipReader(ReaderOptions, this); @@ -98,10 +212,31 @@ public partial class SevenZipArchive : AbstractArchive _database?._packSizes.Aggregate(0L, (total, packSize) => total + packSize) ?? 0; - private sealed class SevenZipReader : AbstractReader + internal sealed class SevenZipReader : AbstractReader { private readonly SevenZipArchive _archive; private SevenZipEntry? _currentEntry; + private Stream? _currentFolderStream; + private CFolder? _currentFolder; + + /// + /// Enables internal diagnostics for tests. + /// When disabled (default), diagnostics properties return null to avoid exposing internal state. + /// + internal bool DiagnosticsEnabled { get; set; } + + /// + /// Current folder instance used to decide whether the solid folder stream should be reused. + /// Only available when is true. + /// + internal object? DiagnosticsCurrentFolder => DiagnosticsEnabled ? _currentFolder : null; + + /// + /// Current shared folder stream instance. + /// Only available when is true. + /// + internal Stream? DiagnosticsCurrentFolderStream => + DiagnosticsEnabled ? _currentFolderStream : null; internal SevenZipReader(ReaderOptions readerOptions, SevenZipArchive archive) : base(readerOptions, ArchiveType.SevenZip) => this._archive = archive; @@ -117,6 +252,10 @@ public partial class SevenZipArchive : AbstractArchive !x.IsDirectory)) { _currentEntry = entry; @@ -131,10 +270,53 @@ public partial class SevenZipArchive : AbstractArchive + /// WORKAROUND: Forces async operations to use synchronous equivalents. + /// This is necessary because the LZMA decoder has bugs in its async implementation + /// that cause state corruption (IndexOutOfRangeException, DataErrorException). + /// + /// The proper fix would be to repair the LZMA decoder's async methods + /// (LzmaStream.ReadAsync, Decoder.CodeAsync, OutWindow async operations), + /// but that requires deep changes to the decoder state machine. + /// private sealed class SyncOnlyStream : Stream { private readonly Stream _baseStream; @@ -164,6 +346,7 @@ public partial class SevenZipArchive : AbstractArchive _baseStream.Write(buffer, offset, count); + // Force async operations to use sync equivalents to avoid LZMA decoder bugs public override Task ReadAsync( byte[] buffer, int offset, diff --git a/src/SharpCompress/Archives/Tar/TarArchive.cs b/src/SharpCompress/Archives/Tar/TarArchive.cs index ded45135..f174f34d 100644 --- a/src/SharpCompress/Archives/Tar/TarArchive.cs +++ b/src/SharpCompress/Archives/Tar/TarArchive.cs @@ -66,7 +66,7 @@ public partial class TarArchive : AbstractWritableArchive + /// The default buffer size for stream operations, matching .NET's Stream.CopyTo default of 81920 bytes. + /// This can be modified globally at runtime. + /// + public static int BufferSize { get; set; } = 81920; +} diff --git a/src/SharpCompress/Factories/ZipFactory.cs b/src/SharpCompress/Factories/ZipFactory.cs index bc75b4ad..5c3fb5f9 100644 --- a/src/SharpCompress/Factories/ZipFactory.cs +++ b/src/SharpCompress/Factories/ZipFactory.cs @@ -49,10 +49,10 @@ public class ZipFactory if (stream is not SharpCompressStream) // wrap to provide buffer bef { - stream = new SharpCompressStream(stream, bufferSize: ReaderOptions.DefaultBufferSize); + stream = new SharpCompressStream(stream, bufferSize: Constants.BufferSize); } - if (ZipArchive.IsZipFile(stream, password, ReaderOptions.DefaultBufferSize)) + if (ZipArchive.IsZipFile(stream, password)) { return true; } @@ -67,63 +67,7 @@ public class ZipFactory stream.Position = startPosition; //test the zip (last) file of a multipart zip - if (ZipArchive.IsZipMulti(stream, password, ReaderOptions.DefaultBufferSize)) - { - return true; - } - - stream.Position = startPosition; - - return false; - } - - /// - public override async ValueTask IsArchiveAsync( - Stream stream, - string? password = null, - CancellationToken cancellationToken = default - ) - { - cancellationToken.ThrowIfCancellationRequested(); - var startPosition = stream.CanSeek ? stream.Position : -1; - - // probe for single volume zip - - if (stream is not SharpCompressStream) // wrap to provide buffer bef - { - stream = new SharpCompressStream(stream, bufferSize: ReaderOptions.DefaultBufferSize); - } - - if ( - await ZipArchive.IsZipFileAsync( - stream, - password, - ReaderOptions.DefaultBufferSize, - cancellationToken - ) - ) - { - return true; - } - - // probe for a multipart zip - - if (!stream.CanSeek) - { - return false; - } - - stream.Position = startPosition; - - //test the zip (last) file of a multipart zip - if ( - await ZipArchive.IsZipMultiAsync( - stream, - password, - ReaderOptions.DefaultBufferSize, - cancellationToken - ) - ) + if (ZipArchive.IsZipMulti(stream, password)) { return true; } diff --git a/src/SharpCompress/IO/SharpCompressStream.Async.cs b/src/SharpCompress/IO/SharpCompressStream.Async.cs index 1e0cbd6c..2b242788 100644 --- a/src/SharpCompress/IO/SharpCompressStream.Async.cs +++ b/src/SharpCompress/IO/SharpCompressStream.Async.cs @@ -24,13 +24,12 @@ public partial class SharpCompressStream { ValidateBufferState(); - // Fill buffer if needed + // Fill buffer if needed, handling short reads from underlying stream if (_bufferedLength == 0) { - _bufferedLength = await Stream - .ReadAsync(_buffer!, 0, _bufferSize, cancellationToken) - .ConfigureAwait(false); _bufferPosition = 0; + _bufferedLength = await FillBufferAsync(_buffer!, 0, _bufferSize, cancellationToken) + .ConfigureAwait(false); } int available = _bufferedLength - _bufferPosition; int toRead = Math.Min(count, available); @@ -42,16 +41,9 @@ public partial class SharpCompressStream return toRead; } // If buffer exhausted, refill - int r = await Stream - .ReadAsync(_buffer!, 0, _bufferSize, cancellationToken) - .ConfigureAwait(false); - if (r == 0) - { - return 0; - } - - _bufferedLength = r; _bufferPosition = 0; + _bufferedLength = await FillBufferAsync(_buffer!, 0, _bufferSize, cancellationToken) + .ConfigureAwait(false); if (_bufferedLength == 0) { return 0; @@ -65,13 +57,47 @@ public partial class SharpCompressStream else { int read = await Stream - .ReadAsync(buffer, offset, count, cancellationToken) - .ConfigureAwait(false); + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); _internalPosition += read; return read; } } + + /// + /// Async version of FillBuffer. Implements the ReadFullyAsync pattern. + /// Reads in a loop until buffer is full or EOF is reached. + /// + private async Task FillBufferAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + // Implement ReadFullyAsync pattern but return the actual count read + // This is the same logic as Utility.ReadFullyAsync but returns count instead of bool + var total = 0; + int read; + while ( + ( + read = await Stream + .ReadAsync(buffer, offset + total, count - total, cancellationToken) + .ConfigureAwait(false) + ) > 0 + ) + { + total += read; + if (total >= count) + { + return total; + } + } + return total; + } + + public override async Task WriteAsync( byte[] buffer, int offset, @@ -104,13 +130,15 @@ public partial class SharpCompressStream { ValidateBufferState(); - // Fill buffer if needed + // Fill buffer if needed, handling short reads from underlying stream if (_bufferedLength == 0) { - _bufferedLength = await Stream - .ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken) - .ConfigureAwait(false); _bufferPosition = 0; + _bufferedLength = await FillBufferMemoryAsync( + _buffer.AsMemory(0, _bufferSize), + cancellationToken + ) + .ConfigureAwait(false); } int available = _bufferedLength - _bufferPosition; int toRead = Math.Min(buffer.Length, available); @@ -122,16 +150,12 @@ public partial class SharpCompressStream return toRead; } // If buffer exhausted, refill - int r = await Stream - .ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken) - .ConfigureAwait(false); - if (r == 0) - { - return 0; - } - - _bufferedLength = r; _bufferPosition = 0; + _bufferedLength = await FillBufferMemoryAsync( + _buffer.AsMemory(0, _bufferSize), + cancellationToken + ) + .ConfigureAwait(false); if (_bufferedLength == 0) { return 0; @@ -150,6 +174,36 @@ public partial class SharpCompressStream } } + /// + /// Async version of FillBuffer for Memory{byte}. Implements the ReadFullyAsync pattern. + /// Reads in a loop until buffer is full or EOF is reached. + /// + private async ValueTask FillBufferMemoryAsync( + Memory buffer, + CancellationToken cancellationToken + ) + { + // Implement ReadFullyAsync pattern but return the actual count read + var total = 0; + int read; + while ( + ( + read = await Stream + .ReadAsync(buffer.Slice(total), cancellationToken) + .ConfigureAwait(false) + ) > 0 + ) + { + total += read; + if (total >= buffer.Length) + { + return total; + } + } + return total; + } + + public override async ValueTask WriteAsync( ReadOnlyMemory buffer, CancellationToken cancellationToken = default diff --git a/src/SharpCompress/IO/SharpCompressStream.cs b/src/SharpCompress/IO/SharpCompressStream.cs index 24f1e51e..58edb522 100644 --- a/src/SharpCompress/IO/SharpCompressStream.cs +++ b/src/SharpCompress/IO/SharpCompressStream.cs @@ -211,11 +211,11 @@ public partial class SharpCompressStream : Stream, IStreamStack { ValidateBufferState(); - // Fill buffer if needed + // Fill buffer if needed, handling short reads from underlying stream if (_bufferedLength == 0) { - _bufferedLength = Stream.Read(_buffer!, 0, _bufferSize); _bufferPosition = 0; + _bufferedLength = FillBuffer(_buffer!, 0, _bufferSize); } int available = _bufferedLength - _bufferPosition; int toRead = Math.Min(count, available); @@ -227,14 +227,8 @@ public partial class SharpCompressStream : Stream, IStreamStack return toRead; } // If buffer exhausted, refill - int r = Stream.Read(_buffer!, 0, _bufferSize); - if (r == 0) - { - return 0; - } - - _bufferedLength = r; _bufferPosition = 0; + _bufferedLength = FillBuffer(_buffer!, 0, _bufferSize); if (_bufferedLength == 0) { return 0; @@ -258,6 +252,31 @@ public partial class SharpCompressStream : Stream, IStreamStack } } + /// + /// Fills the buffer by reading from the underlying stream, handling short reads. + /// Implements the ReadFully pattern: reads in a loop until buffer is full or EOF is reached. + /// + /// Buffer to fill + /// Offset in buffer (always 0 in current usage) + /// Number of bytes to read + /// Total number of bytes read (may be less than count if EOF is reached) + private int FillBuffer(byte[] buffer, int offset, int count) + { + // Implement ReadFully pattern but return the actual count read + // This is the same logic as Utility.ReadFully but returns count instead of bool + var total = 0; + int read; + while ((read = Stream.Read(buffer, offset + total, count - total)) > 0) + { + total += read; + if (total >= count) + { + return total; + } + } + return total; + } + public override long Seek(long offset, SeekOrigin origin) { if (_bufferingEnabled) diff --git a/src/SharpCompress/Readers/AbstractReader.Async.cs b/src/SharpCompress/Readers/AbstractReader.Async.cs index 9dacd88e..958045e0 100644 --- a/src/SharpCompress/Readers/AbstractReader.Async.cs +++ b/src/SharpCompress/Readers/AbstractReader.Async.cs @@ -143,11 +143,15 @@ public abstract partial class AbstractReader #if LEGACY_DOTNET using Stream s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); var sourceStream = WrapWithProgress(s, Entry); - await sourceStream.CopyToAsync(writeStream, 81920, cancellationToken).ConfigureAwait(false); + await sourceStream + .CopyToAsync(writeStream, Constants.BufferSize, cancellationToken) + .ConfigureAwait(false); #else await using Stream s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false); var sourceStream = WrapWithProgress(s, Entry); - await sourceStream.CopyToAsync(writeStream, 81920, cancellationToken).ConfigureAwait(false); + await sourceStream + .CopyToAsync(writeStream, Constants.BufferSize, cancellationToken) + .ConfigureAwait(false); #endif } diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index 18f214a7..7fcc2ab0 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -196,7 +196,7 @@ public abstract partial class AbstractReader : IReader, IAsyncR { using Stream s = OpenEntryStream(); var sourceStream = WrapWithProgress(s, Entry); - sourceStream.CopyTo(writeStream, 81920); + sourceStream.CopyTo(writeStream, Constants.BufferSize); } private Stream WrapWithProgress(Stream source, Entry entry) diff --git a/src/SharpCompress/Readers/ReaderOptions.cs b/src/SharpCompress/Readers/ReaderOptions.cs index cedf8bed..b016a3b6 100644 --- a/src/SharpCompress/Readers/ReaderOptions.cs +++ b/src/SharpCompress/Readers/ReaderOptions.cs @@ -5,6 +5,14 @@ namespace SharpCompress.Readers; public class ReaderOptions : OptionsBase { + /// + /// The default buffer size for stream operations. + /// This value (65536 bytes) is preserved for backward compatibility. + /// New code should use Constants.BufferSize instead (81920 bytes), which matches .NET's Stream.CopyTo default. + /// + [Obsolete( + "Use Constants.BufferSize instead. This constant will be removed in a future version." + )] public const int DefaultBufferSize = 0x10000; /// @@ -16,7 +24,7 @@ public class ReaderOptions : OptionsBase public bool DisableCheckIncomplete { get; set; } - public int BufferSize { get; set; } = DefaultBufferSize; + public int BufferSize { get; set; } = Constants.BufferSize; /// /// Provide a hint for the extension of the archive being read, can speed up finding the correct decoder. Should be without the leading period in the form like: tar.gz or zip diff --git a/src/SharpCompress/Utility.Async.cs b/src/SharpCompress/Utility.Async.cs index be417fd6..99691540 100644 --- a/src/SharpCompress/Utility.Async.cs +++ b/src/SharpCompress/Utility.Async.cs @@ -2,54 +2,122 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.Common; namespace SharpCompress; internal static partial class Utility { - /// - /// Read exactly the requested number of bytes from a stream asynchronously. Throws EndOfStreamException if not enough data is available. - /// - public static async ValueTask ReadExactAsync( - this Stream stream, - byte[] buffer, - int offset, - int length, - CancellationToken cancellationToken = default - ) + extension(Stream source) { - if (stream is null) + /// + /// Read exactly the requested number of bytes from a stream asynchronously. Throws EndOfStreamException if not enough data is available. + /// + public async ValueTask ReadExactAsync( + byte[] buffer, + int offset, + int length, + CancellationToken cancellationToken = default + ) { - throw new ArgumentNullException(nameof(stream)); - } - - if (buffer is null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - if (offset < 0 || offset > buffer.Length) - { - throw new ArgumentOutOfRangeException(nameof(offset)); - } - - if (length < 0 || length > buffer.Length - offset) - { - throw new ArgumentOutOfRangeException(nameof(length)); - } - - while (length > 0) - { - var fetched = await stream - .ReadAsync(buffer, offset, length, cancellationToken) - .ConfigureAwait(false); - if (fetched <= 0) + if (source is null) { - throw new EndOfStreamException(); + throw new ArgumentNullException(nameof(source)); } - offset += fetched; - length -= fetched; + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (offset < 0 || offset > buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + if (length < 0 || length > buffer.Length - offset) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + while (length > 0) + { + var fetched = await source + .ReadAsync(buffer, offset, length, cancellationToken) + .ConfigureAwait(false); + if (fetched <= 0) + { + throw new EndOfStreamException(); + } + + offset += fetched; + length -= fetched; + } + } + + public async ValueTask TransferToAsync( + Stream destination, + long maxLength, + CancellationToken cancellationToken = default + ) + { + // Use ReadOnlySubStream to limit reading and leverage framework's CopyToAsync + using var limitedStream = new IO.ReadOnlySubStream(source, maxLength); + await limitedStream + .CopyToAsync(destination, Constants.BufferSize, cancellationToken) + .ConfigureAwait(false); + return limitedStream.Position; + } + + + public async ValueTask ReadFullyAsync( + byte[] buffer, + CancellationToken cancellationToken = default + ) + { + var total = 0; + int read; + while ( + ( + read = await source + .ReadAsync(buffer, total, buffer.Length - total, cancellationToken) + .ConfigureAwait(false) + ) > 0 + ) + { + total += read; + if (total >= buffer.Length) + { + return true; + } + } + return (total >= buffer.Length); + } + + public async ValueTask ReadFullyAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken = default + ) + { + var total = 0; + int read; + while ( + ( + read = await source + .ReadAsync(buffer, offset + total, count - total, cancellationToken) + .ConfigureAwait(false) + ) > 0 + ) + { + total += read; + if (total >= count) + { + return true; + } + } + return (total >= count); } } } diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index 649e9280..ff2190df 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -6,13 +6,12 @@ using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; +using SharpCompress.Common; namespace SharpCompress; internal static partial class Utility { - //80kb is a good industry standard temporary buffer size - internal const int TEMP_BUFFER_SIZE = 81920; private static readonly HashSet invalidChars = new(Path.GetInvalidFileNameChars()); public static ReadOnlyCollection ToReadOnly(this IList items) => new(items); @@ -135,27 +134,10 @@ internal static partial class Utility { // Use ReadOnlySubStream to limit reading and leverage framework's CopyTo using var limitedStream = new IO.ReadOnlySubStream(source, maxLength); - limitedStream.CopyTo(destination, TEMP_BUFFER_SIZE); + limitedStream.CopyTo(destination, Constants.BufferSize); return limitedStream.Position; } - public async ValueTask TransferToAsync( - Stream destination, - long maxLength, - CancellationToken cancellationToken = default - ) - { - // Use ReadOnlySubStream to limit reading and leverage framework's CopyToAsync - using var limitedStream = new IO.ReadOnlySubStream(source, maxLength); - await limitedStream - .CopyToAsync(destination, TEMP_BUFFER_SIZE, cancellationToken) - .ConfigureAwait(false); - return limitedStream.Position; - } - } - - extension(Stream source) - { public async ValueTask SkipAsync( long advanceAmount, CancellationToken cancellationToken = default @@ -167,19 +149,20 @@ internal static partial class Utility return; } - var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); + var array = ArrayPool.Shared.Rent(Constants.BufferSize); try { while (advanceAmount > 0) { var toRead = (int)Math.Min(array.Length, advanceAmount); var read = await source - .ReadAsync(array, 0, toRead, cancellationToken) - .ConfigureAwait(false); + .ReadAsync(array, 0, toRead, cancellationToken) + .ConfigureAwait(false); if (read <= 0) { break; } + advanceAmount -= read; } } @@ -228,6 +211,7 @@ internal static partial class Utility return true; } } + return (total >= buffer.Length); } @@ -243,100 +227,51 @@ internal static partial class Utility return true; } } + return (total >= buffer.Length); } #endif - public async ValueTask ReadFullyAsync( - byte[] buffer, - CancellationToken cancellationToken = default - ) - { - var total = 0; - int read; - while ( - ( - read = await source - .ReadAsync(buffer, total, buffer.Length - total, cancellationToken) - .ConfigureAwait(false) - ) > 0 - ) - { - total += read; - if (total >= buffer.Length) - { - return true; - } - } - return (total >= buffer.Length); - } - public async ValueTask ReadFullyAsync( - byte[] buffer, - int offset, - int count, - CancellationToken cancellationToken = default - ) + /// + /// Read exactly the requested number of bytes from a stream. Throws EndOfStreamException if not enough data is available. + /// + public void ReadExact(byte[] buffer, int offset, int length) { - var total = 0; - int read; - while ( - ( - read = await source - .ReadAsync(buffer, offset + total, count - total, cancellationToken) - .ConfigureAwait(false) - ) > 0 - ) + if (source is null) { - total += read; - if (total >= count) - { - return true; - } + throw new ArgumentNullException(nameof(source)); + } + + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (offset < 0 || offset > buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + if (length < 0 || length > buffer.Length - offset) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + while (length > 0) + { + var fetched = source.Read(buffer, offset, length); + if (fetched <= 0) + { + throw new EndOfStreamException(); + } + + offset += fetched; + length -= fetched; } - return (total >= count); } } - /// - /// Read exactly the requested number of bytes from a stream. Throws EndOfStreamException if not enough data is available. - /// - public static void ReadExact(this Stream stream, byte[] buffer, int offset, int length) - { - if (stream is null) - { - throw new ArgumentNullException(nameof(stream)); - } - - if (buffer is null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - if (offset < 0 || offset > buffer.Length) - { - throw new ArgumentOutOfRangeException(nameof(offset)); - } - - if (length < 0 || length > buffer.Length - offset) - { - throw new ArgumentOutOfRangeException(nameof(length)); - } - - while (length > 0) - { - var fetched = stream.Read(buffer, offset, length); - if (fetched <= 0) - { - throw new EndOfStreamException(); - } - - offset += fetched; - length -= fetched; - } - } - - // Async methods moved to Utility.Async.cs public static string TrimNulls(this string source) => source.Replace('\0', ' ').Trim(); diff --git a/src/SharpCompress/Writers/GZip/GZipWriter.cs b/src/SharpCompress/Writers/GZip/GZipWriter.cs index 95ebd253..e801b6fc 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriter.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriter.cs @@ -48,7 +48,7 @@ public sealed partial class GZipWriter : AbstractWriter stream.FileName = filename; stream.LastModified = modificationTime; var progressStream = WrapWithProgress(source, filename); - progressStream.CopyTo(stream); + progressStream.CopyTo(stream, Constants.BufferSize); _wroteToStream = true; } diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index be82f1d9..e62ac61d 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -15,6 +15,7 @@ using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.PPMd; using SharpCompress.Compressors.ZStandard; using SharpCompress.IO; +using Constants = SharpCompress.Common.Constants; namespace SharpCompress.Writers.Zip; @@ -87,7 +88,7 @@ public partial class ZipWriter : AbstractWriter { using var output = WriteToStream(entryPath, zipWriterEntryOptions); var progressStream = WrapWithProgress(source, entryPath); - progressStream.CopyTo(output); + progressStream.CopyTo(output, Constants.BufferSize); } public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options) diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index 5e7ece33..41325333 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -216,9 +216,9 @@ "net10.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.2, )", - "resolved": "10.0.2", - "contentHash": "sXdDtMf2qcnbygw9OdE535c2lxSxrZP8gO4UhDJ0xiJbl1wIqXS1OTcTDFTIJPOFd6Mhcm8gPEthqWGUxBsTqw==" + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "kICGrGYEzCNI3wPzfEXcwNHgTvlvVn9yJDhSdRK+oZQy4jvYH529u7O0xf5ocQKzOMjfS07+3z9PKRIjrFMJDA==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", @@ -264,9 +264,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.23, )", - "resolved": "8.0.23", - "contentHash": "GqHiB1HbbODWPbY/lc5xLQH8siEEhNA0ptpJCC6X6adtAYNEzu5ZlqV3YHA3Gh7fuEwgA8XqVwMtH2KNtuQM1Q==" + "requested": "[8.0.22, )", + "resolved": "8.0.22", + "contentHash": "MhcMithKEiyyNkD2ZfbDZPmcOdi0GheGfg8saEIIEfD/fol3iHmcV8TsZkD4ZYz5gdUuoX4YtlVySUU7Sxl9SQ==" }, "Microsoft.NETFramework.ReferenceAssemblies": { "type": "Direct", diff --git a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs index a894606d..2b80ce2d 100644 --- a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs +++ b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs @@ -2,8 +2,8 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; +using SharpCompress.Common; using SharpCompress.IO; -using SharpCompress.Readers; namespace SharpCompress.Test.Mocks; @@ -31,8 +31,8 @@ public class ForwardOnlyStream : SharpCompressStream, IStreamStack public bool IsDisposed { get; private set; } - public ForwardOnlyStream(Stream stream, int bufferSize = ReaderOptions.DefaultBufferSize) - : base(stream, bufferSize: bufferSize) + public ForwardOnlyStream(Stream stream, int? bufferSize = null) + : base(stream, bufferSize: bufferSize ?? Constants.BufferSize) { this.stream = stream; #if DEBUG_STREAMS diff --git a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs index 2a7163e8..da9e0965 100644 --- a/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs +++ b/tests/SharpCompress.Test/SevenZip/SevenZipArchiveTests.cs @@ -253,4 +253,98 @@ public class SevenZipArchiveTests : ArchiveTests ); Assert.False(nonSolidArchive.IsSolid); } + + [Fact] + public void SevenZipArchive_Solid_ExtractAllEntries_Contiguous() + { + // This test verifies that solid archives iterate entries as contiguous streams + // rather than recreating the decompression stream for each entry + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.7z"); + using var archive = SevenZipArchive.Open(testArchive); + Assert.True(archive.IsSolid); + + using var reader = archive.ExtractAllEntries(); + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + reader.WriteEntryToDirectory( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + + VerifyFiles(); + } + + [Fact] + public void SevenZipArchive_Solid_VerifyStreamReuse() + { + // This test verifies that the folder stream is reused within each folder + // and not recreated for each entry in solid archives + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.7z"); + using var archive = SevenZipArchive.Open(testArchive); + Assert.True(archive.IsSolid); + + using var reader = archive.ExtractAllEntries(); + + var sevenZipReader = Assert.IsType(reader); + sevenZipReader.DiagnosticsEnabled = true; + + Stream? currentFolderStreamInstance = null; + object? currentFolder = null; + var entryCount = 0; + var entriesInCurrentFolder = 0; + var streamRecreationsWithinFolder = 0; + + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + // Extract the entry to trigger GetEntryStream + using var entryStream = reader.OpenEntryStream(); + var buffer = new byte[4096]; + while (entryStream.Read(buffer, 0, buffer.Length) > 0) + { + // Read the stream to completion + } + + entryCount++; + + var folderStream = sevenZipReader.DiagnosticsCurrentFolderStream; + var folder = sevenZipReader.DiagnosticsCurrentFolder; + + Assert.NotNull(folderStream); // Folder stream should exist + + // Check if we're in a new folder + if (currentFolder == null || !ReferenceEquals(currentFolder, folder)) + { + // Starting a new folder + currentFolder = folder; + currentFolderStreamInstance = folderStream; + entriesInCurrentFolder = 1; + } + else + { + // Same folder - verify stream wasn't recreated + entriesInCurrentFolder++; + + if (!ReferenceEquals(currentFolderStreamInstance, folderStream)) + { + // Stream was recreated within the same folder - this is the bug we're testing for! + streamRecreationsWithinFolder++; + } + + currentFolderStreamInstance = folderStream; + } + } + } + + // Verify we actually tested multiple entries + Assert.True(entryCount > 1, "Test should have multiple entries to verify stream reuse"); + + // The critical check: within a single folder, the stream should NEVER be recreated + Assert.Equal(0, streamRecreationsWithinFolder); // Folder stream should remain the same for all entries in the same folder + } } diff --git a/tests/SharpCompress.Test/Zip/ZipShortReadTests.cs b/tests/SharpCompress.Test/Zip/ZipShortReadTests.cs new file mode 100644 index 00000000..d61a27c7 --- /dev/null +++ b/tests/SharpCompress.Test/Zip/ZipShortReadTests.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test.Zip; + +/// +/// Tests for ZIP reading with streams that return short reads. +/// Reproduces the regression where ZIP parsing fails depending on Stream.Read chunking patterns. +/// +public class ZipShortReadTests : ReaderTests +{ + /// + /// A non-seekable stream that returns controlled short reads. + /// Simulates real-world network/multipart streams that legally return fewer bytes than requested. + /// + private sealed class PatternReadStream : Stream + { + private readonly MemoryStream _inner; + private readonly int _firstReadSize; + private readonly int _chunkSize; + private bool _firstReadDone; + + public PatternReadStream(byte[] bytes, int firstReadSize, int chunkSize) + { + _inner = new MemoryStream(bytes, writable: false); + _firstReadSize = firstReadSize; + _chunkSize = chunkSize; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int limit = !_firstReadDone ? _firstReadSize : _chunkSize; + _firstReadDone = true; + + int toRead = Math.Min(count, limit); + return _inner.Read(buffer, offset, toRead); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + } + + /// + /// Test that ZIP reading works correctly with short reads on non-seekable streams. + /// Uses a test archive and different chunking patterns. + /// + [Theory] + [InlineData("Zip.deflate.zip", 1000, 4096)] + [InlineData("Zip.deflate.zip", 999, 4096)] + [InlineData("Zip.deflate.zip", 100, 4096)] + [InlineData("Zip.deflate.zip", 50, 512)] + [InlineData("Zip.deflate.zip", 1, 1)] // Extreme case: 1 byte at a time + [InlineData("Zip.deflate.dd.zip", 1000, 4096)] + [InlineData("Zip.deflate.dd.zip", 999, 4096)] + [InlineData("Zip.zip64.zip", 3816, 4096)] + [InlineData("Zip.zip64.zip", 3815, 4096)] // Similar to the issue pattern + public void Zip_Reader_Handles_Short_Reads(string zipFile, int firstReadSize, int chunkSize) + { + // Use an existing test ZIP file + var zipPath = Path.Combine(TEST_ARCHIVES_PATH, zipFile); + if (!File.Exists(zipPath)) + { + return; // Skip if file doesn't exist + } + + var bytes = File.ReadAllBytes(zipPath); + + // Baseline with MemoryStream (seekable, no short reads) + var baseline = ReadEntriesFromStream(new MemoryStream(bytes, writable: false)); + Assert.NotEmpty(baseline); + + // Non-seekable stream with controlled short read pattern + var chunked = ReadEntriesFromStream(new PatternReadStream(bytes, firstReadSize, chunkSize)); + Assert.Equal(baseline, chunked); + } + + private List ReadEntriesFromStream(Stream stream) + { + var names = new List(); + using var reader = ReaderFactory.Open(stream, new ReaderOptions { LeaveStreamOpen = true }); + + while (reader.MoveToNextEntry()) + { + if (reader.Entry.IsDirectory) + { + continue; + } + + names.Add(reader.Entry.Key!); + + using var entryStream = reader.OpenEntryStream(); + entryStream.CopyTo(Stream.Null); + } + + return names; + } +} diff --git a/tests/SharpCompress.Test/packages.lock.json b/tests/SharpCompress.Test/packages.lock.json index 78330583..a2139072 100644 --- a/tests/SharpCompress.Test/packages.lock.json +++ b/tests/SharpCompress.Test/packages.lock.json @@ -319,30 +319,6 @@ } } }, - ".NETFramework,Version=v4.8/win-x86": { - "Microsoft.Win32.Registry": { - "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", - "dependencies": { - "System.Security.AccessControl": "5.0.0", - "System.Security.Principal.Windows": "5.0.0" - } - }, - "System.Security.AccessControl": { - "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", - "dependencies": { - "System.Security.Principal.Windows": "5.0.0" - } - }, - "System.Security.Principal.Windows": { - "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" - } - }, "net10.0": { "AwesomeAssertions": { "type": "Direct", @@ -555,13 +531,6 @@ "resolved": "10.0.0", "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==" } - }, - "net10.0/win-x86": { - "Microsoft.Win32.Registry": { - "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" - } } } } \ No newline at end of file