Add WriterOptions.BufferSize property

This commit is contained in:
Adam Hathcock
2026-06-12 14:41:05 +01:00
parent 3a73cfe925
commit de6e2bfee2
19 changed files with 210 additions and 29 deletions

View File

@@ -92,7 +92,8 @@ using (var archive = GZipArchive.CreateArchive())
// With fluent options (preferred)
var options = WriterOptions.ForZip()
.WithCompressionLevel(9)
.WithLeaveStreamOpen(false);
.WithLeaveStreamOpen(false)
.WithBufferSize(131072);
using (var archive = ZipArchive.CreateArchive())
{
archive.SaveTo("output.zip", options);
@@ -102,10 +103,13 @@ using (var archive = ZipArchive.CreateArchive())
var options2 = new WriterOptions(CompressionType.Deflate)
{
CompressionLevel = 9,
LeaveStreamOpen = false
LeaveStreamOpen = false,
BufferSize = 131072
};
```
`WriterOptions.BufferSize` controls stream copy buffers used while writing archive entries. If it is not set, SharpCompress falls back to `Constants.BufferSize`.
---
## Archive API Methods

View File

@@ -17,7 +17,8 @@ public static class IArchiveEntryExtensions
/// </summary>
/// <param name="streamToWriteTo">The stream to write the entry content to.</param>
/// <param name="progress">Optional progress reporter for tracking extraction progress.</param>
public void WriteTo(Stream streamToWriteTo, IProgress<ProgressReport>? progress = null) => archiveEntry.WriteTo(streamToWriteTo, null, progress);
public void WriteTo(Stream streamToWriteTo, IProgress<ProgressReport>? progress = null) =>
archiveEntry.WriteTo(streamToWriteTo, null, progress);
private void WriteTo(
Stream streamToWriteTo,

View File

@@ -8,6 +8,7 @@ public static class Constants
/// The default buffer size for stream operations, matching .NET's Stream.CopyTo default of 81920 bytes.
/// This can be modified globally at runtime.
/// </summary>
// TODO: Revisit remaining non-extraction usages after extraction buffering moves to ExtractionOptions.
public static int BufferSize { get; set; } = 81920;
/// <summary>

View File

@@ -19,6 +19,11 @@ public interface IWriterOptions : IStreamOptions, IEncodingOptions, IProgressOpt
/// </summary>
int CompressionLevel { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
int BufferSize { get; set; }
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom providers, such as

View File

@@ -1,7 +1,9 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.IO;
namespace SharpCompress.Readers;
@@ -42,7 +44,8 @@ public static class IAsyncReaderExtensions
async (x, fm, ct) =>
{
using var fs = File.Open(x, fm);
await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false);
await CopyEntryToAsync(reader, fs, options?.BufferSize, ct)
.ConfigureAwait(false);
},
cancellationToken
)
@@ -77,7 +80,8 @@ public static class IAsyncReaderExtensions
async (x, fm, ct) =>
{
using var fs = File.Open(x, fm);
await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false);
await CopyEntryToAsync(reader, fs, options?.BufferSize, ct)
.ConfigureAwait(false);
},
cancellationToken
)
@@ -92,4 +96,58 @@ public static class IAsyncReaderExtensions
.WriteEntryToAsync(destinationFileInfo.FullName, options, cancellationToken)
.ConfigureAwait(false);
}
private static async ValueTask CopyEntryToAsync(
IAsyncReader reader,
Stream writableStream,
int? bufferSize,
CancellationToken cancellationToken
)
{
#if LEGACY_DOTNET
using var entryStream = await reader
.OpenEntryStreamAsync(cancellationToken)
.ConfigureAwait(false);
#else
await using var entryStream = await reader
.OpenEntryStreamAsync(cancellationToken)
.ConfigureAwait(false);
#endif
var sourceStream = WrapWithProgress(entryStream, reader.Entry);
await sourceStream
.CopyToAsync(writableStream, bufferSize ?? Constants.BufferSize, cancellationToken)
.ConfigureAwait(false);
}
private static Stream WrapWithProgress(Stream source, IEntry entry)
{
var progress = entry.Options.Progress;
if (progress is null)
{
return source;
}
var entryPath = entry.Key ?? string.Empty;
var totalBytes = GetEntrySizeSafe(entry);
return new ProgressReportingStream(
source,
progress,
entryPath,
totalBytes,
leaveOpen: true
);
}
private static long? GetEntrySizeSafe(IEntry entry)
{
try
{
var size = entry.Size;
return size >= 0 ? size : null;
}
catch (NotImplementedException)
{
return null;
}
}
}

View File

@@ -65,13 +65,14 @@ internal static partial class Utility
public async ValueTask<long> TransferToAsync(
Stream destination,
long maxLength,
int? bufferSize = null,
CancellationToken cancellationToken = default
)
{
// Use ReadOnlySubStream to limit reading and leverage framework's CopyToAsync
using var limitedStream = new IO.ReadOnlySubStream(source, maxLength);
await limitedStream
.CopyToAsync(destination, Constants.BufferSize, cancellationToken)
.CopyToAsync(destination, bufferSize ?? Constants.BufferSize, cancellationToken)
.ConfigureAwait(false);
return limitedStream.Position;
}

View File

@@ -150,11 +150,11 @@ internal static partial class Utility
extension(Stream source)
{
public long TransferTo(Stream destination, long maxLength)
public long TransferTo(Stream destination, long maxLength, int? bufferSize)
{
// Use ReadOnlySubStream to limit reading and leverage framework's CopyTo
using var limitedStream = new IO.ReadOnlySubStream(source, maxLength);
limitedStream.CopyTo(destination, Constants.BufferSize);
limitedStream.CopyTo(destination, bufferSize ?? Constants.BufferSize);
return limitedStream.Position;
}

View File

@@ -86,7 +86,7 @@ public sealed partial class GZipWriter : AbstractWriter
}
var progressStream = WrapWithProgress(source, filename);
progressStream.CopyTo(OutputStream.NotNull(), Constants.BufferSize);
progressStream.CopyTo(OutputStream.NotNull(), WriterOptions.BufferSize);
_wroteToStream = true;
}

View File

@@ -68,6 +68,11 @@ public sealed record GZipWriterOptions : IWriterOptions
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations, such as
@@ -115,6 +120,7 @@ public sealed record GZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}
@@ -128,6 +134,7 @@ public sealed record GZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}
}

View File

@@ -57,6 +57,11 @@ public sealed record SevenZipWriterOptions : IWriterOptions
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
@@ -103,6 +108,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}
@@ -117,6 +123,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}

View File

@@ -104,7 +104,12 @@ public partial class TarWriter
await header.WriteAsync(OutputStream.NotNull(), cancellationToken).ConfigureAwait(false);
var progressStream = WrapWithProgress(source, filename);
var written = await progressStream
.TransferToAsync(OutputStream.NotNull(), realSize, cancellationToken)
.TransferToAsync(
OutputStream.NotNull(),
realSize,
WriterOptions.BufferSize,
cancellationToken
)
.ConfigureAwait(false);
await PadTo512Async(written, cancellationToken).ConfigureAwait(false);
}

View File

@@ -133,7 +133,11 @@ public partial class TarWriter : AbstractWriter
header.Size = realSize;
header.Write(OutputStream.NotNull());
var progressStream = WrapWithProgress(source, filename);
size = progressStream.TransferTo(OutputStream.NotNull(), realSize);
size = progressStream.TransferTo(
OutputStream.NotNull(),
realSize,
WriterOptions.BufferSize
);
PadTo512(size.Value);
}

View File

@@ -44,6 +44,11 @@ public sealed record TarWriterOptions : IWriterOptions
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
@@ -104,6 +109,7 @@ public sealed record TarWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}
@@ -118,6 +124,7 @@ public sealed record TarWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}

View File

@@ -56,6 +56,11 @@ public sealed record WriterOptions : IWriterOptions
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations, such as

View File

@@ -53,6 +53,15 @@ public static class WriterOptionsExtensions
),
};
/// <summary>
/// Creates a copy with the specified buffer size.
/// </summary>
public static WriterOptions WithBufferSize(this WriterOptions options, int bufferSize) =>
options with
{
BufferSize = bufferSize,
};
/// <summary>
/// Creates a copy with the specified compression level.
/// </summary>

View File

@@ -16,7 +16,6 @@ using SharpCompress.Compressors.PPMd;
using SharpCompress.Compressors.ZStandard;
using SharpCompress.IO;
using SharpCompress.Providers;
using Constants = SharpCompress.Common.Constants;
namespace SharpCompress.Writers.Zip;
@@ -89,7 +88,7 @@ public partial class ZipWriter : AbstractWriter
{
using var output = WriteToStream(entryPath, zipWriterEntryOptions);
var progressStream = WrapWithProgress(source, entryPath);
progressStream.CopyTo(output, Constants.BufferSize);
progressStream.CopyTo(output, WriterOptions.BufferSize);
}
public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options)

View File

@@ -61,6 +61,11 @@ public sealed record ZipWriterOptions : IWriterOptions
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Buffer size for writer stream copy operations.
/// </summary>
public int BufferSize { get; set; } = Constants.BufferSize;
/// <summary>
/// Registry of compression providers.
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
@@ -128,6 +133,7 @@ public sealed record ZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}
@@ -141,6 +147,7 @@ public sealed record ZipWriterOptions : IWriterOptions
LeaveStreamOpen = options.LeaveStreamOpen;
ArchiveEncoding = options.ArchiveEncoding;
Progress = options.Progress;
BufferSize = options.BufferSize;
Providers = options.Providers;
}

View File

@@ -11,6 +11,7 @@ using SharpCompress.Readers;
using SharpCompress.Test.Mocks;
using SharpCompress.Writers;
using SharpCompress.Writers.GZip;
using SharpCompress.Writers.Tar;
using SharpCompress.Writers.Zip;
using Xunit;
@@ -133,11 +134,16 @@ public class OptionsUsabilityTests : TestBase
[Fact]
public void WriterOptions_Fluent_Methods_Modify_Correctly()
{
var options = WriterOptions.ForZip().WithLeaveStreamOpen(false).WithCompressionLevel(9);
var options = WriterOptions
.ForZip()
.WithLeaveStreamOpen(false)
.WithCompressionLevel(9)
.WithBufferSize(65536);
Assert.Equal(CompressionType.Deflate, options.CompressionType);
Assert.Equal(9, options.CompressionLevel);
Assert.False(options.LeaveStreamOpen);
Assert.Equal(65536, options.BufferSize);
}
[Fact]
@@ -161,6 +167,73 @@ public class OptionsUsabilityTests : TestBase
Assert.Equal(factoryApproach.LeaveStreamOpen, constructorApproach.LeaveStreamOpen);
}
[Fact]
public void WriterOptions_Default_BufferSize_Uses_Constants_BufferSize()
{
Assert.Equal(Constants.BufferSize, WriterOptions.ForZip().BufferSize);
Assert.Equal(
Constants.BufferSize,
new ZipWriterOptions(CompressionType.Deflate).BufferSize
);
Assert.Equal(
Constants.BufferSize,
new TarWriterOptions(CompressionType.None, true).BufferSize
);
Assert.Equal(Constants.BufferSize, new GZipWriterOptions().BufferSize);
}
[Fact]
public void Format_WriterOptions_Copy_BufferSize()
{
var options = WriterOptions.ForZip().WithBufferSize(12345);
Assert.Equal(12345, new ZipWriterOptions(options).BufferSize);
Assert.Equal(12345, new TarWriterOptions(options).BufferSize);
Assert.Equal(12345, new GZipWriterOptions(options).BufferSize);
}
[Fact]
public void ZipWriter_Uses_WriterOptions_BufferSize()
{
using var source = new TrackingReadStream(new byte[100]);
using var destination = new MemoryStream();
using var writer = new ZipWriter(
destination,
new ZipWriterOptions(CompressionType.None) { BufferSize = 17 }
);
writer.Write("buffer-size.txt", source, DateTime.Now);
Assert.Equal(17, source.CopyBufferSize);
}
[Fact]
public void GZipWriter_Uses_WriterOptions_BufferSize()
{
using var source = new TrackingReadStream(new byte[100]);
using var destination = new MemoryStream();
using var writer = new GZipWriter(destination, new GZipWriterOptions { BufferSize = 19 });
writer.Write("buffer-size.txt", source, DateTime.Now);
Assert.Equal(19, source.CopyBufferSize);
}
[Fact]
public void TarWriter_Uses_WriterOptions_BufferSize()
{
using var source = new TrackingReadStream(new byte[100]);
using var destination = new MemoryStream();
using var writer = new TarWriter(
destination,
new TarWriterOptions(CompressionType.None, true) { BufferSize = 0 }
);
Assert.Throws<ArgumentOutOfRangeException>(() =>
writer.Write("buffer-size.txt", source, DateTime.Now)
);
}
[Fact]
public void ReaderOptions_Fluent_Methods_Modify_Correctly()
{
@@ -375,12 +448,6 @@ public class OptionsUsabilityTests : TestBase
{
public int? CopyBufferSize { get; private set; }
public override void CopyTo(Stream destination, int bufferSize)
{
CopyBufferSize = bufferSize;
base.CopyTo(destination, bufferSize);
}
public override Task CopyToAsync(
Stream destination,
int bufferSize,
@@ -477,12 +544,6 @@ public class OptionsUsabilityTests : TestBase
Action<int> copyBufferSize
) : EntryStream(reader, stream)
{
public override void CopyTo(Stream destination, int bufferSize)
{
copyBufferSize(bufferSize);
base.CopyTo(destination, bufferSize);
}
public override Task CopyToAsync(
Stream destination,
int bufferSize,

View File

@@ -579,7 +579,7 @@ public class UtilityTests
using var source = new MemoryStream(sourceData);
using var destination = new MemoryStream();
var transferred = source.TransferTo(destination, 5);
var transferred = source.TransferTo(destination, 5, null);
Assert.Equal(5, transferred);
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, destination.ToArray());
@@ -592,7 +592,7 @@ public class UtilityTests
using var source = new MemoryStream(sourceData);
using var destination = new MemoryStream();
var transferred = source.TransferTo(destination, 100);
var transferred = source.TransferTo(destination, 100, null);
Assert.Equal(3, transferred);
Assert.Equal(sourceData, destination.ToArray());
@@ -604,7 +604,7 @@ public class UtilityTests
using var source = new MemoryStream();
using var destination = new MemoryStream();
var transferred = source.TransferTo(destination, 100);
var transferred = source.TransferTo(destination, 100, null);
Assert.Equal(0, transferred);
Assert.Empty(destination.ToArray());