Compare commits

...

27 Commits

Author SHA1 Message Date
Adam Hathcock
9a7bdd39e8 Merge pull request #1172 from adamhathcock/copilot/fix-sevenzip-contiguous-streams
Fix SevenZipReader to maintain contiguous stream state for solid archives
2026-01-28 08:35:28 +00:00
Adam Hathcock
484bc740d7 Update src/SharpCompress/Archives/SevenZip/SevenZipArchive.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-28 08:26:28 +00:00
Adam Hathcock
8a67d501a8 Don't use reflection in tests 2026-01-28 08:10:06 +00:00
copilot-swe-agent[bot]
3c87242bd0 Add test to verify folder stream reuse in solid archives
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 17:29:44 +00:00
copilot-swe-agent[bot]
999124e68e Remove unused _currentFolderIndex field
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 17:03:20 +00:00
copilot-swe-agent[bot]
db2f5c9cb9 Fix SevenZipReader to iterate entries as contiguous streams
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 17:01:18 +00:00
Adam Hathcock
af08a7cd54 Merge pull request #1169 from adamhathcock/copilot/fix-zip-parsing-regression
Fix ZIP parsing failure on non-seekable streams with short reads
2026-01-27 16:54:12 +00:00
copilot-swe-agent[bot]
72eaf66f05 Initial plan 2026-01-27 16:53:53 +00:00
Adam Hathcock
8a3be35d67 Update tests/SharpCompress.Test/Zip/ZipShortReadTests.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-27 16:43:13 +00:00
copilot-swe-agent[bot]
d59e4c2a0d Refactor FillBuffer to use ReadFully pattern
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 16:25:24 +00:00
copilot-swe-agent[bot]
71655e04c4 Apply code formatting with CSharpier
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 16:02:26 +00:00
copilot-swe-agent[bot]
a706a9d725 Fix ZIP parsing regression with short reads on non-seekable streams
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 16:00:44 +00:00
copilot-swe-agent[bot]
970934a40b Initial plan 2026-01-27 15:51:50 +00:00
Adam Hathcock
a9c28a7b62 Merge pull request #1165 from adamhathcock/adam/buffer-size-consolidation
(Release) Buffer size consolidation
2026-01-27 14:41:14 +00:00
Adam Hathcock
4d31436740 constant should be a static property 2026-01-27 12:39:01 +00:00
Adam Hathcock
c82744c51c fmt 2026-01-27 12:15:31 +00:00
Adam Hathcock
f0eaddc6a6 Merge remote-tracking branch 'origin/adam/buffer-size-consolidation' into adam/buffer-size-consolidation 2026-01-27 12:14:17 +00:00
Adam Hathcock
d6156f0f1e release branch builds increment patch versions and master builds increment minor versions 2026-01-27 12:14:03 +00:00
Adam Hathcock
3c88c7fdd5 Merge pull request #1167 from adamhathcock/copilot/sub-pr-1165-again
Fix grammatical errors in ArcFactory comment documentation
2026-01-27 11:58:25 +00:00
Adam Hathcock
d11f6aefb0 Merge pull request #1166 from adamhathcock/copilot/sub-pr-1165
Add [Obsolete] attribute to ReaderOptions.DefaultBufferSize for backward compatibility
2026-01-27 11:57:54 +00:00
copilot-swe-agent[bot]
010a38bb73 Add clarifying comment about buffer size value difference
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 11:54:50 +00:00
copilot-swe-agent[bot]
53f12d75db Add [Obsolete] attribute to ReaderOptions.DefaultBufferSize
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 11:53:37 +00:00
copilot-swe-agent[bot]
6c866324b2 Fix grammatical errors in ArcFactory comments
Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>
2026-01-27 11:49:29 +00:00
copilot-swe-agent[bot]
a114155189 Initial plan 2026-01-27 11:48:05 +00:00
copilot-swe-agent[bot]
014bbc3ea4 Initial plan 2026-01-27 11:47:52 +00:00
Adam Hathcock
d52facd4ab Remove change 2026-01-27 10:48:32 +00:00
Adam Hathcock
0a50386ada Using Constants class differently 2026-01-27 10:46:54 +00:00
28 changed files with 495 additions and 159 deletions

View File

@@ -230,7 +230,7 @@ static async Task<(string version, bool isPrerelease)> GetVersion()
}
else
{
// Not tagged - create prerelease version based on next minor version
// Not tagged - create prerelease version
var allTags = (await GetGitOutput("tag", "--list"))
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
.Where(tag => Regex.IsMatch(tag.Trim(), @"^\d+\.\d+\.\d+$"))
@@ -240,8 +240,22 @@ static async Task<(string version, bool isPrerelease)> GetVersion()
var lastTag = allTags.OrderBy(tag => Version.Parse(tag)).LastOrDefault() ?? "0.0.0";
var lastVersion = Version.Parse(lastTag);
// Increment minor version for next release
var nextVersion = new Version(lastVersion.Major, lastVersion.Minor + 1, 0);
// Determine version increment based on branch
var currentBranch = await GetCurrentBranch();
Version nextVersion;
if (currentBranch == "release")
{
// Release branch: increment patch version
nextVersion = new Version(lastVersion.Major, lastVersion.Minor, lastVersion.Build + 1);
Console.WriteLine($"Building prerelease for release branch (patch increment)");
}
else
{
// Master or other branches: increment minor version
nextVersion = new Version(lastVersion.Major, lastVersion.Minor + 1, 0);
Console.WriteLine($"Building prerelease for {currentBranch} branch (minor increment)");
}
// Use commit count since the last version tag if available; otherwise, fall back to total count
var revListArgs = allTags.Any() ? $"--count {lastTag}..HEAD" : "--count HEAD";
@@ -253,6 +267,28 @@ static async Task<(string version, bool isPrerelease)> GetVersion()
}
}
static async Task<string> GetCurrentBranch()
{
// In GitHub Actions, GITHUB_REF_NAME contains the branch name
var githubRefName = Environment.GetEnvironmentVariable("GITHUB_REF_NAME");
if (!string.IsNullOrEmpty(githubRefName))
{
return githubRefName;
}
// Fallback to git command for local builds
try
{
var (output, _) = await ReadAsync("git", "branch --show-current");
return output.Trim();
}
catch (Exception ex)
{
Console.WriteLine($"Warning: Could not determine current branch: {ex.Message}");
return "unknown";
}
}
static async Task<string> GetGitOutput(string command, string args)
{
try

View File

@@ -166,22 +166,14 @@ public static class ArchiveFactory
);
}
public static bool IsArchive(
string filePath,
out ArchiveType? type,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public static bool IsArchive(string filePath, out ArchiveType? type)
{
filePath.NotNullOrEmpty(nameof(filePath));
using Stream s = File.OpenRead(filePath);
return IsArchive(s, out type, bufferSize);
return IsArchive(s, out type);
}
public static bool IsArchive(
Stream stream,
out ArchiveType? type,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public static bool IsArchive(Stream stream, out ArchiveType? type)
{
type = null;
stream.NotNull(nameof(stream));

View File

@@ -14,11 +14,8 @@ class AutoArchiveFactory : IArchiveFactory
public IEnumerable<string> GetSupportedExtensions() => throw new NotSupportedException();
public bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => throw new NotSupportedException();
public bool IsArchive(Stream stream, string? password = null) =>
throw new NotSupportedException();
public FileInfo? GetFilePart(int index, FileInfo part1) => throw new NotSupportedException();

View File

@@ -9,8 +9,6 @@ namespace SharpCompress.Archives;
public static class IArchiveEntryExtensions
{
private const int BufferSize = 81920;
/// <param name="archiveEntry">The archive entry to extract.</param>
extension(IArchiveEntry archiveEntry)
{
@@ -28,7 +26,7 @@ public static class IArchiveEntryExtensions
using var entryStream = archiveEntry.OpenEntryStream();
var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress);
sourceStream.CopyTo(streamToWriteTo, BufferSize);
sourceStream.CopyTo(streamToWriteTo, Constants.BufferSize);
}
/// <summary>
@@ -51,7 +49,7 @@ public static class IArchiveEntryExtensions
using var entryStream = await archiveEntry.OpenEntryStreamAsync(cancellationToken);
var sourceStream = WrapWithProgress(entryStream, archiveEntry, progress);
await sourceStream
.CopyToAsync(streamToWriteTo, BufferSize, cancellationToken)
.CopyToAsync(streamToWriteTo, Constants.BufferSize, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -212,10 +212,31 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
public override long TotalSize =>
_database?._packSizes.Aggregate(0L, (total, packSize) => total + packSize) ?? 0;
private sealed class SevenZipReader : AbstractReader<SevenZipEntry, SevenZipVolume>
internal sealed class SevenZipReader : AbstractReader<SevenZipEntry, SevenZipVolume>
{
private readonly SevenZipArchive _archive;
private SevenZipEntry? _currentEntry;
private Stream? _currentFolderStream;
private CFolder? _currentFolder;
/// <summary>
/// Enables internal diagnostics for tests.
/// When disabled (default), diagnostics properties return null to avoid exposing internal state.
/// </summary>
internal bool DiagnosticsEnabled { get; set; }
/// <summary>
/// Current folder instance used to decide whether the solid folder stream should be reused.
/// Only available when <see cref="DiagnosticsEnabled"/> is true.
/// </summary>
internal object? DiagnosticsCurrentFolder => DiagnosticsEnabled ? _currentFolder : null;
/// <summary>
/// Current shared folder stream instance.
/// Only available when <see cref="DiagnosticsEnabled"/> is true.
/// </summary>
internal Stream? DiagnosticsCurrentFolderStream =>
DiagnosticsEnabled ? _currentFolderStream : null;
internal SevenZipReader(ReaderOptions readerOptions, SevenZipArchive archive)
: base(readerOptions, ArchiveType.SevenZip) => this._archive = archive;
@@ -231,9 +252,10 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
_currentEntry = dir;
yield return dir;
}
// For non-directory entries, yield them without creating shared streams
// Each call to GetEntryStream() will create a fresh decompression stream
// to avoid state corruption issues with async operations
// For solid archives (entries in the same folder share a compressed stream),
// we must iterate entries sequentially and maintain the folder stream state
// across entries in the same folder to avoid recreating the decompression
// stream for each file, which breaks contiguous streaming.
foreach (var entry in entries.Where(x => !x.IsDirectory))
{
_currentEntry = entry;
@@ -243,19 +265,53 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
protected override EntryStream GetEntryStream()
{
// Create a fresh decompression stream for each file (no state sharing).
// However, the LZMA decoder has bugs in its async implementation that cause
// state corruption even on fresh streams. The SyncOnlyStream wrapper
// works around these bugs by forcing async operations to use sync equivalents.
//
// TODO: Fix the LZMA decoder async bugs (in LzmaStream, Decoder, OutWindow)
// so this wrapper is no longer necessary.
var entry = _currentEntry.NotNull("currentEntry is not null");
if (entry.IsDirectory)
{
return CreateEntryStream(Stream.Null);
}
return CreateEntryStream(new SyncOnlyStream(entry.FilePart.GetCompressedStream()));
var filePart = (SevenZipFilePart)entry.FilePart;
if (!filePart.Header.HasStream)
{
// Entries with no underlying stream (e.g., empty files or anti-items)
// should return an empty stream, matching previous behavior.
return CreateEntryStream(Stream.Null);
}
var folder = filePart.Folder;
// Check if we're starting a new folder - dispose old folder stream if needed
if (folder != _currentFolder)
{
_currentFolderStream?.Dispose();
_currentFolderStream = null;
_currentFolder = folder;
}
// Create the folder stream once per folder
if (_currentFolderStream is null)
{
_currentFolderStream = _archive._database!.GetFolderStream(
_archive.Volumes.Single().Stream,
folder!,
_archive._database.PasswordProvider
);
}
// Wrap with SyncOnlyStream to work around LZMA async bugs
// Return a ReadOnlySubStream that reads from the shared folder stream
return CreateEntryStream(
new SyncOnlyStream(
new ReadOnlySubStream(_currentFolderStream, entry.Size, leaveOpen: true)
)
);
}
public override void Dispose()
{
_currentFolderStream?.Dispose();
_currentFolderStream = null;
base.Dispose();
}
}

View File

@@ -180,7 +180,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
using (var entryStream = entry.OpenEntryStream())
{
using var memoryStream = new MemoryStream();
entryStream.CopyTo(memoryStream);
entryStream.CopyTo(memoryStream, Constants.BufferSize);
memoryStream.Position = 0;
var bytes = memoryStream.ToArray();

View File

@@ -124,38 +124,27 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
);
}
public static bool IsZipFile(
string filePath,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => IsZipFile(new FileInfo(filePath), password, bufferSize);
public static bool IsZipFile(string filePath, string? password = null) =>
IsZipFile(new FileInfo(filePath), password);
public static bool IsZipFile(
FileInfo fileInfo,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public static bool IsZipFile(FileInfo fileInfo, string? password = null)
{
if (!fileInfo.Exists)
{
return false;
}
using Stream stream = fileInfo.OpenRead();
return IsZipFile(stream, password, bufferSize);
return IsZipFile(stream, password);
}
public static bool IsZipFile(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public static bool IsZipFile(Stream stream, string? password = null)
{
var headerFactory = new StreamingZipHeaderFactory(password, new ArchiveEncoding(), null);
try
{
if (stream is not SharpCompressStream)
{
stream = new SharpCompressStream(stream, bufferSize: bufferSize);
stream = new SharpCompressStream(stream, bufferSize: Constants.BufferSize);
}
var header = headerFactory
@@ -177,18 +166,14 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
}
}
public static bool IsZipMulti(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public static bool IsZipMulti(Stream stream, string? password = null)
{
var headerFactory = new StreamingZipHeaderFactory(password, new ArchiveEncoding(), null);
try
{
if (stream is not SharpCompressStream)
{
stream = new SharpCompressStream(stream, bufferSize: bufferSize);
stream = new SharpCompressStream(stream, bufferSize: Constants.BufferSize);
}
var header = headerFactory
@@ -229,7 +214,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
if (streams.Count() > 1) //test part 2 - true = multipart not split
{
streams[1].Position += 4; //skip the POST_DATA_DESCRIPTOR to prevent an exception
var isZip = IsZipFile(streams[1], ReaderOptions.Password, ReaderOptions.BufferSize);
var isZip = IsZipFile(streams[1], ReaderOptions.Password);
streams[1].Position -= 4;
if (isZip)
{

View File

@@ -0,0 +1,10 @@
namespace SharpCompress.Common;
public static class Constants
{
/// <summary>
/// The default buffer size for stream operations, matching .NET's Stream.CopyTo default of 81920 bytes.
/// This can be modified globally at runtime.
/// </summary>
public static int BufferSize { get; set; } = 81920;
}

View File

@@ -22,11 +22,7 @@ namespace SharpCompress.Factories
yield return "ace";
}
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public override bool IsArchive(Stream stream, string? password = null)
{
return AceHeader.IsArchive(stream);
}

View File

@@ -23,11 +23,7 @@ namespace SharpCompress.Factories
yield return "arc";
}
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public override bool IsArchive(Stream stream, string? password = null)
{
//You may have to use some(paranoid) checks to ensure that you actually are
//processing an ARC file, since other archivers also adopted the idea of putting

View File

@@ -22,11 +22,7 @@ namespace SharpCompress.Factories
yield return "arj";
}
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public override bool IsArchive(Stream stream, string? password = null)
{
return ArjHeader.IsArchive(stream);
}

View File

@@ -51,11 +51,7 @@ public abstract class Factory : IFactory
public abstract IEnumerable<string> GetSupportedExtensions();
/// <inheritdoc/>
public abstract bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
);
public abstract bool IsArchive(Stream stream, string? password = null);
/// <inheritdoc/>
public virtual FileInfo? GetFilePart(int index, FileInfo part1) => null;
@@ -82,7 +78,7 @@ public abstract class Factory : IFactory
{
long pos = ((IStreamStack)stream).GetPosition();
if (IsArchive(stream, options.Password, options.BufferSize))
if (IsArchive(stream, options.Password))
{
((IStreamStack)stream).StackSeek(pos);
reader = readerFactory.OpenReader(stream, options);

View File

@@ -40,11 +40,8 @@ public class GZipFactory
}
/// <inheritdoc/>
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => GZipArchive.IsGZipFile(stream);
public override bool IsArchive(Stream stream, string? password = null) =>
GZipArchive.IsGZipFile(stream);
#endregion

View File

@@ -36,11 +36,7 @@ public interface IFactory
/// </summary>
/// <param name="stream">A stream, pointing to the beginning of the archive.</param>
/// <param name="password">optional password</param>
bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
);
bool IsArchive(Stream stream, string? password = null);
/// <summary>
/// From a passed in archive (zip, rar, 7z, 001), return all parts.

View File

@@ -29,11 +29,8 @@ public class RarFactory : Factory, IArchiveFactory, IMultiArchiveFactory, IReade
}
/// <inheritdoc/>
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => RarArchive.IsRarFile(stream);
public override bool IsArchive(Stream stream, string? password = null) =>
RarArchive.IsRarFile(stream);
/// <inheritdoc/>
public override FileInfo? GetFilePart(int index, FileInfo part1) =>

View File

@@ -28,11 +28,8 @@ public class SevenZipFactory : Factory, IArchiveFactory, IMultiArchiveFactory
}
/// <inheritdoc/>
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => SevenZipArchive.IsSevenZipFile(stream);
public override bool IsArchive(Stream stream, string? password = null) =>
SevenZipArchive.IsSevenZipFile(stream);
#endregion

View File

@@ -53,11 +53,8 @@ public class TarFactory
}
/// <inheritdoc/>
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
) => TarArchive.IsTarFile(stream);
public override bool IsArchive(Stream stream, string? password = null) =>
TarArchive.IsTarFile(stream);
#endregion

View File

@@ -20,9 +20,6 @@ internal class ZStandardFactory : Factory
yield return "zstd";
}
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = 65536
) => ZStandardStream.IsZStandard(stream);
public override bool IsArchive(Stream stream, string? password = null) =>
ZStandardStream.IsZStandard(stream);
}

View File

@@ -39,11 +39,7 @@ public class ZipFactory
}
/// <inheritdoc/>
public override bool IsArchive(
Stream stream,
string? password = null,
int bufferSize = ReaderOptions.DefaultBufferSize
)
public override bool IsArchive(Stream stream, string? password = null)
{
var startPosition = stream.CanSeek ? stream.Position : -1;
@@ -51,10 +47,10 @@ public class ZipFactory
if (stream is not SharpCompressStream) // wrap to provide buffer bef
{
stream = new SharpCompressStream(stream, bufferSize: bufferSize);
stream = new SharpCompressStream(stream, bufferSize: Constants.BufferSize);
}
if (ZipArchive.IsZipFile(stream, password, bufferSize))
if (ZipArchive.IsZipFile(stream, password))
{
return true;
}
@@ -69,7 +65,7 @@ public class ZipFactory
stream.Position = startPosition;
//test the zip (last) file of a multipart zip
if (ZipArchive.IsZipMulti(stream, password, bufferSize))
if (ZipArchive.IsZipMulti(stream, password))
{
return true;
}

View File

@@ -206,11 +206,11 @@ public class SharpCompressStream : Stream, IStreamStack
{
ValidateBufferState();
// Fill buffer if needed
// Fill buffer if needed, handling short reads from underlying stream
if (_bufferedLength == 0)
{
_bufferedLength = Stream.Read(_buffer!, 0, _bufferSize);
_bufferPosition = 0;
_bufferedLength = FillBuffer(_buffer!, 0, _bufferSize);
}
int available = _bufferedLength - _bufferPosition;
int toRead = Math.Min(count, available);
@@ -222,11 +222,8 @@ public class SharpCompressStream : Stream, IStreamStack
return toRead;
}
// If buffer exhausted, refill
int r = Stream.Read(_buffer!, 0, _bufferSize);
if (r == 0)
return 0;
_bufferedLength = r;
_bufferPosition = 0;
_bufferedLength = FillBuffer(_buffer!, 0, _bufferSize);
if (_bufferedLength == 0)
{
return 0;
@@ -250,6 +247,31 @@ public class SharpCompressStream : Stream, IStreamStack
}
}
/// <summary>
/// Fills the buffer by reading from the underlying stream, handling short reads.
/// Implements the ReadFully pattern: reads in a loop until buffer is full or EOF is reached.
/// </summary>
/// <param name="buffer">Buffer to fill</param>
/// <param name="offset">Offset in buffer (always 0 in current usage)</param>
/// <param name="count">Number of bytes to read</param>
/// <returns>Total number of bytes read (may be less than count if EOF is reached)</returns>
private int FillBuffer(byte[] buffer, int offset, int count)
{
// Implement ReadFully pattern but return the actual count read
// This is the same logic as Utility.ReadFully but returns count instead of bool
var total = 0;
int read;
while ((read = Stream.Read(buffer, offset + total, count - total)) > 0)
{
total += read;
if (total >= count)
{
return total;
}
}
return total;
}
public override long Seek(long offset, SeekOrigin origin)
{
if (_bufferingEnabled)
@@ -324,13 +346,12 @@ public class SharpCompressStream : Stream, IStreamStack
{
ValidateBufferState();
// Fill buffer if needed
// Fill buffer if needed, handling short reads from underlying stream
if (_bufferedLength == 0)
{
_bufferedLength = await Stream
.ReadAsync(_buffer!, 0, _bufferSize, cancellationToken)
.ConfigureAwait(false);
_bufferPosition = 0;
_bufferedLength = await FillBufferAsync(_buffer!, 0, _bufferSize, cancellationToken)
.ConfigureAwait(false);
}
int available = _bufferedLength - _bufferPosition;
int toRead = Math.Min(count, available);
@@ -342,13 +363,9 @@ public class SharpCompressStream : Stream, IStreamStack
return toRead;
}
// If buffer exhausted, refill
int r = await Stream
.ReadAsync(_buffer!, 0, _bufferSize, cancellationToken)
.ConfigureAwait(false);
if (r == 0)
return 0;
_bufferedLength = r;
_bufferPosition = 0;
_bufferedLength = await FillBufferAsync(_buffer!, 0, _bufferSize, cancellationToken)
.ConfigureAwait(false);
if (_bufferedLength == 0)
{
return 0;
@@ -369,6 +386,38 @@ public class SharpCompressStream : Stream, IStreamStack
}
}
/// <summary>
/// Async version of FillBuffer. Implements the ReadFullyAsync pattern.
/// Reads in a loop until buffer is full or EOF is reached.
/// </summary>
private async Task<int> FillBufferAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
// Implement ReadFullyAsync pattern but return the actual count read
// This is the same logic as Utility.ReadFullyAsync but returns count instead of bool
var total = 0;
int read;
while (
(
read = await Stream
.ReadAsync(buffer, offset + total, count - total, cancellationToken)
.ConfigureAwait(false)
) > 0
)
{
total += read;
if (total >= count)
{
return total;
}
}
return total;
}
public override async Task WriteAsync(
byte[] buffer,
int offset,
@@ -399,13 +448,15 @@ public class SharpCompressStream : Stream, IStreamStack
{
ValidateBufferState();
// Fill buffer if needed
// Fill buffer if needed, handling short reads from underlying stream
if (_bufferedLength == 0)
{
_bufferedLength = await Stream
.ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken)
.ConfigureAwait(false);
_bufferPosition = 0;
_bufferedLength = await FillBufferMemoryAsync(
_buffer.AsMemory(0, _bufferSize),
cancellationToken
)
.ConfigureAwait(false);
}
int available = _bufferedLength - _bufferPosition;
int toRead = Math.Min(buffer.Length, available);
@@ -417,13 +468,12 @@ public class SharpCompressStream : Stream, IStreamStack
return toRead;
}
// If buffer exhausted, refill
int r = await Stream
.ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken)
.ConfigureAwait(false);
if (r == 0)
return 0;
_bufferedLength = r;
_bufferPosition = 0;
_bufferedLength = await FillBufferMemoryAsync(
_buffer.AsMemory(0, _bufferSize),
cancellationToken
)
.ConfigureAwait(false);
if (_bufferedLength == 0)
{
return 0;
@@ -442,6 +492,35 @@ public class SharpCompressStream : Stream, IStreamStack
}
}
/// <summary>
/// Async version of FillBuffer for Memory{byte}. Implements the ReadFullyAsync pattern.
/// Reads in a loop until buffer is full or EOF is reached.
/// </summary>
private async ValueTask<int> FillBufferMemoryAsync(
Memory<byte> buffer,
CancellationToken cancellationToken
)
{
// Implement ReadFullyAsync pattern but return the actual count read
var total = 0;
int read;
while (
(
read = await Stream
.ReadAsync(buffer.Slice(total), cancellationToken)
.ConfigureAwait(false)
) > 0
)
{
total += read;
if (total >= buffer.Length)
{
return total;
}
}
return total;
}
public override async ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default

View File

@@ -262,7 +262,7 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
{
using Stream s = OpenEntryStream();
var sourceStream = WrapWithProgress(s, Entry);
sourceStream.CopyTo(writeStream, 81920);
sourceStream.CopyTo(writeStream, Constants.BufferSize);
}
internal async Task WriteAsync(Stream writeStream, CancellationToken cancellationToken)
@@ -270,11 +270,15 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader
#if NETFRAMEWORK || NETSTANDARD2_0
using Stream s = OpenEntryStream();
var sourceStream = WrapWithProgress(s, Entry);
await sourceStream.CopyToAsync(writeStream, 81920, cancellationToken).ConfigureAwait(false);
await sourceStream
.CopyToAsync(writeStream, Constants.BufferSize, cancellationToken)
.ConfigureAwait(false);
#else
await using Stream s = OpenEntryStream();
var sourceStream = WrapWithProgress(s, Entry);
await sourceStream.CopyToAsync(writeStream, 81920, cancellationToken).ConfigureAwait(false);
await sourceStream
.CopyToAsync(writeStream, Constants.BufferSize, cancellationToken)
.ConfigureAwait(false);
#endif
}

View File

@@ -5,6 +5,14 @@ namespace SharpCompress.Readers;
public class ReaderOptions : OptionsBase
{
/// <summary>
/// The default buffer size for stream operations.
/// This value (65536 bytes) is preserved for backward compatibility.
/// New code should use Constants.BufferSize instead (81920 bytes), which matches .NET's Stream.CopyTo default.
/// </summary>
[Obsolete(
"Use Constants.BufferSize instead. This constant will be removed in a future version."
)]
public const int DefaultBufferSize = 0x10000;
/// <summary>
@@ -16,7 +24,7 @@ public class ReaderOptions : OptionsBase
public bool DisableCheckIncomplete { get; set; }
public int BufferSize { get; set; } = DefaultBufferSize;
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Provide a hint for the extension of the archive being read, can speed up finding the correct decoder. Should be without the leading period in the form like: tar.gz or zip

View File

@@ -11,8 +11,6 @@ namespace SharpCompress;
internal static class Utility
{
//80kb is a good industry standard temporary buffer size
private const int TEMP_BUFFER_SIZE = 81920;
private static readonly HashSet<char> invalidChars = new(Path.GetInvalidFileNameChars());
public static ReadOnlyCollection<T> ToReadOnly<T>(this IList<T> items) => new(items);
@@ -151,7 +149,7 @@ internal static class Utility
public static long TransferTo(this Stream source, Stream destination, long maxLength)
{
var array = ArrayPool<byte>.Shared.Rent(TEMP_BUFFER_SIZE);
var array = ArrayPool<byte>.Shared.Rent(Common.Constants.BufferSize);
try
{
var maxReadSize = array.Length;
@@ -190,7 +188,7 @@ internal static class Utility
CancellationToken cancellationToken = default
)
{
var array = ArrayPool<byte>.Shared.Rent(TEMP_BUFFER_SIZE);
var array = ArrayPool<byte>.Shared.Rent(Common.Constants.BufferSize);
try
{
var maxReadSize = array.Length;
@@ -268,7 +266,7 @@ internal static class Utility
return;
}
var array = ArrayPool<byte>.Shared.Rent(TEMP_BUFFER_SIZE);
var array = ArrayPool<byte>.Shared.Rent(Common.Constants.BufferSize);
try
{
while (advanceAmount > 0)

View File

@@ -48,7 +48,7 @@ public sealed class GZipWriter : AbstractWriter
stream.FileName = filename;
stream.LastModified = modificationTime;
var progressStream = WrapWithProgress(source, filename);
progressStream.CopyTo(stream);
progressStream.CopyTo(stream, Constants.BufferSize);
_wroteToStream = true;
}

View File

@@ -15,6 +15,7 @@ using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.PPMd;
using SharpCompress.Compressors.ZStandard;
using SharpCompress.IO;
using Constants = SharpCompress.Common.Constants;
namespace SharpCompress.Writers.Zip;
@@ -87,7 +88,7 @@ public class ZipWriter : AbstractWriter
{
using var output = WriteToStream(entryPath, zipWriterEntryOptions);
var progressStream = WrapWithProgress(source, entryPath);
progressStream.CopyTo(output);
progressStream.CopyTo(output, Constants.BufferSize);
}
public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options)

View File

@@ -2,8 +2,8 @@ using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.IO;
using SharpCompress.Readers;
namespace SharpCompress.Test.Mocks;
@@ -31,8 +31,8 @@ public class ForwardOnlyStream : SharpCompressStream, IStreamStack
public bool IsDisposed { get; private set; }
public ForwardOnlyStream(Stream stream, int bufferSize = ReaderOptions.DefaultBufferSize)
: base(stream, bufferSize: bufferSize)
public ForwardOnlyStream(Stream stream, int? bufferSize = null)
: base(stream, bufferSize: bufferSize ?? Constants.BufferSize)
{
this.stream = stream;
#if DEBUG_STREAMS

View File

@@ -251,4 +251,98 @@ public class SevenZipArchiveTests : ArchiveTests
);
Assert.False(nonSolidArchive.IsSolid);
}
[Fact]
public void SevenZipArchive_Solid_ExtractAllEntries_Contiguous()
{
// This test verifies that solid archives iterate entries as contiguous streams
// rather than recreating the decompression stream for each entry
var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.7z");
using var archive = SevenZipArchive.Open(testArchive);
Assert.True(archive.IsSolid);
using var reader = archive.ExtractAllEntries();
while (reader.MoveToNextEntry())
{
if (!reader.Entry.IsDirectory)
{
reader.WriteEntryToDirectory(
SCRATCH_FILES_PATH,
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
);
}
}
VerifyFiles();
}
[Fact]
public void SevenZipArchive_Solid_VerifyStreamReuse()
{
// This test verifies that the folder stream is reused within each folder
// and not recreated for each entry in solid archives
var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "7Zip.solid.7z");
using var archive = SevenZipArchive.Open(testArchive);
Assert.True(archive.IsSolid);
using var reader = archive.ExtractAllEntries();
var sevenZipReader = Assert.IsType<SevenZipArchive.SevenZipReader>(reader);
sevenZipReader.DiagnosticsEnabled = true;
Stream? currentFolderStreamInstance = null;
object? currentFolder = null;
var entryCount = 0;
var entriesInCurrentFolder = 0;
var streamRecreationsWithinFolder = 0;
while (reader.MoveToNextEntry())
{
if (!reader.Entry.IsDirectory)
{
// Extract the entry to trigger GetEntryStream
using var entryStream = reader.OpenEntryStream();
var buffer = new byte[4096];
while (entryStream.Read(buffer, 0, buffer.Length) > 0)
{
// Read the stream to completion
}
entryCount++;
var folderStream = sevenZipReader.DiagnosticsCurrentFolderStream;
var folder = sevenZipReader.DiagnosticsCurrentFolder;
Assert.NotNull(folderStream); // Folder stream should exist
// Check if we're in a new folder
if (currentFolder == null || !ReferenceEquals(currentFolder, folder))
{
// Starting a new folder
currentFolder = folder;
currentFolderStreamInstance = folderStream;
entriesInCurrentFolder = 1;
}
else
{
// Same folder - verify stream wasn't recreated
entriesInCurrentFolder++;
if (!ReferenceEquals(currentFolderStreamInstance, folderStream))
{
// Stream was recreated within the same folder - this is the bug we're testing for!
streamRecreationsWithinFolder++;
}
currentFolderStreamInstance = folderStream;
}
}
}
// Verify we actually tested multiple entries
Assert.True(entryCount > 1, "Test should have multiple entries to verify stream reuse");
// The critical check: within a single folder, the stream should NEVER be recreated
Assert.Equal(0, streamRecreationsWithinFolder); // Folder stream should remain the same for all entries in the same folder
}
}

View File

@@ -0,0 +1,117 @@
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Readers;
using Xunit;
namespace SharpCompress.Test.Zip;
/// <summary>
/// Tests for ZIP reading with streams that return short reads.
/// Reproduces the regression where ZIP parsing fails depending on Stream.Read chunking patterns.
/// </summary>
public class ZipShortReadTests : ReaderTests
{
/// <summary>
/// A non-seekable stream that returns controlled short reads.
/// Simulates real-world network/multipart streams that legally return fewer bytes than requested.
/// </summary>
private sealed class PatternReadStream : Stream
{
private readonly MemoryStream _inner;
private readonly int _firstReadSize;
private readonly int _chunkSize;
private bool _firstReadDone;
public PatternReadStream(byte[] bytes, int firstReadSize, int chunkSize)
{
_inner = new MemoryStream(bytes, writable: false);
_firstReadSize = firstReadSize;
_chunkSize = chunkSize;
}
public override int Read(byte[] buffer, int offset, int count)
{
int limit = !_firstReadDone ? _firstReadSize : _chunkSize;
_firstReadDone = true;
int toRead = Math.Min(count, limit);
return _inner.Read(buffer, offset, toRead);
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Flush() => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
}
/// <summary>
/// Test that ZIP reading works correctly with short reads on non-seekable streams.
/// Uses a test archive and different chunking patterns.
/// </summary>
[Theory]
[InlineData("Zip.deflate.zip", 1000, 4096)]
[InlineData("Zip.deflate.zip", 999, 4096)]
[InlineData("Zip.deflate.zip", 100, 4096)]
[InlineData("Zip.deflate.zip", 50, 512)]
[InlineData("Zip.deflate.zip", 1, 1)] // Extreme case: 1 byte at a time
[InlineData("Zip.deflate.dd.zip", 1000, 4096)]
[InlineData("Zip.deflate.dd.zip", 999, 4096)]
[InlineData("Zip.zip64.zip", 3816, 4096)]
[InlineData("Zip.zip64.zip", 3815, 4096)] // Similar to the issue pattern
public void Zip_Reader_Handles_Short_Reads(string zipFile, int firstReadSize, int chunkSize)
{
// Use an existing test ZIP file
var zipPath = Path.Combine(TEST_ARCHIVES_PATH, zipFile);
if (!File.Exists(zipPath))
{
return; // Skip if file doesn't exist
}
var bytes = File.ReadAllBytes(zipPath);
// Baseline with MemoryStream (seekable, no short reads)
var baseline = ReadEntriesFromStream(new MemoryStream(bytes, writable: false));
Assert.NotEmpty(baseline);
// Non-seekable stream with controlled short read pattern
var chunked = ReadEntriesFromStream(new PatternReadStream(bytes, firstReadSize, chunkSize));
Assert.Equal(baseline, chunked);
}
private List<string> ReadEntriesFromStream(Stream stream)
{
var names = new List<string>();
using var reader = ReaderFactory.Open(stream, new ReaderOptions { LeaveStreamOpen = true });
while (reader.MoveToNextEntry())
{
if (reader.Entry.IsDirectory)
{
continue;
}
names.Add(reader.Entry.Key!);
using var entryStream = reader.OpenEntryStream();
entryStream.CopyTo(Stream.Null);
}
return names;
}
}