mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-07-08 18:16:30 +00:00
Add WriterOptions.BufferSize property
This commit is contained in:
@@ -92,7 +92,8 @@ using (var archive = GZipArchive.CreateArchive())
|
||||
// With fluent options (preferred)
|
||||
var options = WriterOptions.ForZip()
|
||||
.WithCompressionLevel(9)
|
||||
.WithLeaveStreamOpen(false);
|
||||
.WithLeaveStreamOpen(false)
|
||||
.WithBufferSize(131072);
|
||||
using (var archive = ZipArchive.CreateArchive())
|
||||
{
|
||||
archive.SaveTo("output.zip", options);
|
||||
@@ -102,10 +103,13 @@ using (var archive = ZipArchive.CreateArchive())
|
||||
var options2 = new WriterOptions(CompressionType.Deflate)
|
||||
{
|
||||
CompressionLevel = 9,
|
||||
LeaveStreamOpen = false
|
||||
LeaveStreamOpen = false,
|
||||
BufferSize = 131072
|
||||
};
|
||||
```
|
||||
|
||||
`WriterOptions.BufferSize` controls stream copy buffers used while writing archive entries. If it is not set, SharpCompress falls back to `Constants.BufferSize`.
|
||||
|
||||
---
|
||||
|
||||
## Archive API Methods
|
||||
|
||||
@@ -17,7 +17,8 @@ public static class IArchiveEntryExtensions
|
||||
/// </summary>
|
||||
/// <param name="streamToWriteTo">The stream to write the entry content to.</param>
|
||||
/// <param name="progress">Optional progress reporter for tracking extraction progress.</param>
|
||||
public void WriteTo(Stream streamToWriteTo, IProgress<ProgressReport>? progress = null) => archiveEntry.WriteTo(streamToWriteTo, null, progress);
|
||||
public void WriteTo(Stream streamToWriteTo, IProgress<ProgressReport>? progress = null) =>
|
||||
archiveEntry.WriteTo(streamToWriteTo, null, progress);
|
||||
|
||||
private void WriteTo(
|
||||
Stream streamToWriteTo,
|
||||
|
||||
@@ -8,6 +8,7 @@ public static class Constants
|
||||
/// The default buffer size for stream operations, matching .NET's Stream.CopyTo default of 81920 bytes.
|
||||
/// This can be modified globally at runtime.
|
||||
/// </summary>
|
||||
// TODO: Revisit remaining non-extraction usages after extraction buffering moves to ExtractionOptions.
|
||||
public static int BufferSize { get; set; } = 81920;
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -19,6 +19,11 @@ public interface IWriterOptions : IStreamOptions, IEncodingOptions, IProgressOpt
|
||||
/// </summary>
|
||||
int CompressionLevel { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
int BufferSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom providers, such as
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.IO;
|
||||
|
||||
namespace SharpCompress.Readers;
|
||||
|
||||
@@ -42,7 +44,8 @@ public static class IAsyncReaderExtensions
|
||||
async (x, fm, ct) =>
|
||||
{
|
||||
using var fs = File.Open(x, fm);
|
||||
await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false);
|
||||
await CopyEntryToAsync(reader, fs, options?.BufferSize, ct)
|
||||
.ConfigureAwait(false);
|
||||
},
|
||||
cancellationToken
|
||||
)
|
||||
@@ -77,7 +80,8 @@ public static class IAsyncReaderExtensions
|
||||
async (x, fm, ct) =>
|
||||
{
|
||||
using var fs = File.Open(x, fm);
|
||||
await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false);
|
||||
await CopyEntryToAsync(reader, fs, options?.BufferSize, ct)
|
||||
.ConfigureAwait(false);
|
||||
},
|
||||
cancellationToken
|
||||
)
|
||||
@@ -92,4 +96,58 @@ public static class IAsyncReaderExtensions
|
||||
.WriteEntryToAsync(destinationFileInfo.FullName, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async ValueTask CopyEntryToAsync(
|
||||
IAsyncReader reader,
|
||||
Stream writableStream,
|
||||
int? bufferSize,
|
||||
CancellationToken cancellationToken
|
||||
)
|
||||
{
|
||||
#if LEGACY_DOTNET
|
||||
using var entryStream = await reader
|
||||
.OpenEntryStreamAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
#else
|
||||
await using var entryStream = await reader
|
||||
.OpenEntryStreamAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
#endif
|
||||
var sourceStream = WrapWithProgress(entryStream, reader.Entry);
|
||||
await sourceStream
|
||||
.CopyToAsync(writableStream, bufferSize ?? Constants.BufferSize, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Stream WrapWithProgress(Stream source, IEntry entry)
|
||||
{
|
||||
var progress = entry.Options.Progress;
|
||||
if (progress is null)
|
||||
{
|
||||
return source;
|
||||
}
|
||||
|
||||
var entryPath = entry.Key ?? string.Empty;
|
||||
var totalBytes = GetEntrySizeSafe(entry);
|
||||
return new ProgressReportingStream(
|
||||
source,
|
||||
progress,
|
||||
entryPath,
|
||||
totalBytes,
|
||||
leaveOpen: true
|
||||
);
|
||||
}
|
||||
|
||||
private static long? GetEntrySizeSafe(IEntry entry)
|
||||
{
|
||||
try
|
||||
{
|
||||
var size = entry.Size;
|
||||
return size >= 0 ? size : null;
|
||||
}
|
||||
catch (NotImplementedException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,13 +65,14 @@ internal static partial class Utility
|
||||
public async ValueTask<long> TransferToAsync(
|
||||
Stream destination,
|
||||
long maxLength,
|
||||
int? bufferSize = null,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
// Use ReadOnlySubStream to limit reading and leverage framework's CopyToAsync
|
||||
using var limitedStream = new IO.ReadOnlySubStream(source, maxLength);
|
||||
await limitedStream
|
||||
.CopyToAsync(destination, Constants.BufferSize, cancellationToken)
|
||||
.CopyToAsync(destination, bufferSize ?? Constants.BufferSize, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return limitedStream.Position;
|
||||
}
|
||||
|
||||
@@ -150,11 +150,11 @@ internal static partial class Utility
|
||||
|
||||
extension(Stream source)
|
||||
{
|
||||
public long TransferTo(Stream destination, long maxLength)
|
||||
public long TransferTo(Stream destination, long maxLength, int? bufferSize)
|
||||
{
|
||||
// Use ReadOnlySubStream to limit reading and leverage framework's CopyTo
|
||||
using var limitedStream = new IO.ReadOnlySubStream(source, maxLength);
|
||||
limitedStream.CopyTo(destination, Constants.BufferSize);
|
||||
limitedStream.CopyTo(destination, bufferSize ?? Constants.BufferSize);
|
||||
return limitedStream.Position;
|
||||
}
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ public sealed partial class GZipWriter : AbstractWriter
|
||||
}
|
||||
|
||||
var progressStream = WrapWithProgress(source, filename);
|
||||
progressStream.CopyTo(OutputStream.NotNull(), Constants.BufferSize);
|
||||
progressStream.CopyTo(OutputStream.NotNull(), WriterOptions.BufferSize);
|
||||
_wroteToStream = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -68,6 +68,11 @@ public sealed record GZipWriterOptions : IWriterOptions
|
||||
/// </summary>
|
||||
public IProgress<ProgressReport>? Progress { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
public int BufferSize { get; set; } = Constants.BufferSize;
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations, such as
|
||||
@@ -115,6 +120,7 @@ public sealed record GZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
@@ -128,6 +134,7 @@ public sealed record GZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,11 @@ public sealed record SevenZipWriterOptions : IWriterOptions
|
||||
/// </summary>
|
||||
public IProgress<ProgressReport>? Progress { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
public int BufferSize { get; set; } = Constants.BufferSize;
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
|
||||
@@ -103,6 +108,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
@@ -117,6 +123,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
|
||||
@@ -104,7 +104,12 @@ public partial class TarWriter
|
||||
await header.WriteAsync(OutputStream.NotNull(), cancellationToken).ConfigureAwait(false);
|
||||
var progressStream = WrapWithProgress(source, filename);
|
||||
var written = await progressStream
|
||||
.TransferToAsync(OutputStream.NotNull(), realSize, cancellationToken)
|
||||
.TransferToAsync(
|
||||
OutputStream.NotNull(),
|
||||
realSize,
|
||||
WriterOptions.BufferSize,
|
||||
cancellationToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
await PadTo512Async(written, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -133,7 +133,11 @@ public partial class TarWriter : AbstractWriter
|
||||
header.Size = realSize;
|
||||
header.Write(OutputStream.NotNull());
|
||||
var progressStream = WrapWithProgress(source, filename);
|
||||
size = progressStream.TransferTo(OutputStream.NotNull(), realSize);
|
||||
size = progressStream.TransferTo(
|
||||
OutputStream.NotNull(),
|
||||
realSize,
|
||||
WriterOptions.BufferSize
|
||||
);
|
||||
PadTo512(size.Value);
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,11 @@ public sealed record TarWriterOptions : IWriterOptions
|
||||
/// </summary>
|
||||
public IProgress<ProgressReport>? Progress { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
public int BufferSize { get; set; } = Constants.BufferSize;
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
|
||||
@@ -104,6 +109,7 @@ public sealed record TarWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
@@ -118,6 +124,7 @@ public sealed record TarWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,11 @@ public sealed record WriterOptions : IWriterOptions
|
||||
/// </summary>
|
||||
public IProgress<ProgressReport>? Progress { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
public int BufferSize { get; set; } = Constants.BufferSize;
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations, such as
|
||||
|
||||
@@ -53,6 +53,15 @@ public static class WriterOptionsExtensions
|
||||
),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a copy with the specified buffer size.
|
||||
/// </summary>
|
||||
public static WriterOptions WithBufferSize(this WriterOptions options, int bufferSize) =>
|
||||
options with
|
||||
{
|
||||
BufferSize = bufferSize,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a copy with the specified compression level.
|
||||
/// </summary>
|
||||
|
||||
@@ -16,7 +16,6 @@ using SharpCompress.Compressors.PPMd;
|
||||
using SharpCompress.Compressors.ZStandard;
|
||||
using SharpCompress.IO;
|
||||
using SharpCompress.Providers;
|
||||
using Constants = SharpCompress.Common.Constants;
|
||||
|
||||
namespace SharpCompress.Writers.Zip;
|
||||
|
||||
@@ -89,7 +88,7 @@ public partial class ZipWriter : AbstractWriter
|
||||
{
|
||||
using var output = WriteToStream(entryPath, zipWriterEntryOptions);
|
||||
var progressStream = WrapWithProgress(source, entryPath);
|
||||
progressStream.CopyTo(output, Constants.BufferSize);
|
||||
progressStream.CopyTo(output, WriterOptions.BufferSize);
|
||||
}
|
||||
|
||||
public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options)
|
||||
|
||||
@@ -61,6 +61,11 @@ public sealed record ZipWriterOptions : IWriterOptions
|
||||
/// </summary>
|
||||
public IProgress<ProgressReport>? Progress { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer size for writer stream copy operations.
|
||||
/// </summary>
|
||||
public int BufferSize { get; set; } = Constants.BufferSize;
|
||||
|
||||
/// <summary>
|
||||
/// Registry of compression providers.
|
||||
/// Defaults to <see cref="CompressionProviderRegistry.Default" /> but can be replaced with custom implementations.
|
||||
@@ -128,6 +133,7 @@ public sealed record ZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
@@ -141,6 +147,7 @@ public sealed record ZipWriterOptions : IWriterOptions
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
Progress = options.Progress;
|
||||
BufferSize = options.BufferSize;
|
||||
Providers = options.Providers;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ using SharpCompress.Readers;
|
||||
using SharpCompress.Test.Mocks;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.GZip;
|
||||
using SharpCompress.Writers.Tar;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
@@ -133,11 +134,16 @@ public class OptionsUsabilityTests : TestBase
|
||||
[Fact]
|
||||
public void WriterOptions_Fluent_Methods_Modify_Correctly()
|
||||
{
|
||||
var options = WriterOptions.ForZip().WithLeaveStreamOpen(false).WithCompressionLevel(9);
|
||||
var options = WriterOptions
|
||||
.ForZip()
|
||||
.WithLeaveStreamOpen(false)
|
||||
.WithCompressionLevel(9)
|
||||
.WithBufferSize(65536);
|
||||
|
||||
Assert.Equal(CompressionType.Deflate, options.CompressionType);
|
||||
Assert.Equal(9, options.CompressionLevel);
|
||||
Assert.False(options.LeaveStreamOpen);
|
||||
Assert.Equal(65536, options.BufferSize);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -161,6 +167,73 @@ public class OptionsUsabilityTests : TestBase
|
||||
Assert.Equal(factoryApproach.LeaveStreamOpen, constructorApproach.LeaveStreamOpen);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WriterOptions_Default_BufferSize_Uses_Constants_BufferSize()
|
||||
{
|
||||
Assert.Equal(Constants.BufferSize, WriterOptions.ForZip().BufferSize);
|
||||
Assert.Equal(
|
||||
Constants.BufferSize,
|
||||
new ZipWriterOptions(CompressionType.Deflate).BufferSize
|
||||
);
|
||||
Assert.Equal(
|
||||
Constants.BufferSize,
|
||||
new TarWriterOptions(CompressionType.None, true).BufferSize
|
||||
);
|
||||
Assert.Equal(Constants.BufferSize, new GZipWriterOptions().BufferSize);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Format_WriterOptions_Copy_BufferSize()
|
||||
{
|
||||
var options = WriterOptions.ForZip().WithBufferSize(12345);
|
||||
|
||||
Assert.Equal(12345, new ZipWriterOptions(options).BufferSize);
|
||||
Assert.Equal(12345, new TarWriterOptions(options).BufferSize);
|
||||
Assert.Equal(12345, new GZipWriterOptions(options).BufferSize);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZipWriter_Uses_WriterOptions_BufferSize()
|
||||
{
|
||||
using var source = new TrackingReadStream(new byte[100]);
|
||||
using var destination = new MemoryStream();
|
||||
using var writer = new ZipWriter(
|
||||
destination,
|
||||
new ZipWriterOptions(CompressionType.None) { BufferSize = 17 }
|
||||
);
|
||||
|
||||
writer.Write("buffer-size.txt", source, DateTime.Now);
|
||||
|
||||
Assert.Equal(17, source.CopyBufferSize);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GZipWriter_Uses_WriterOptions_BufferSize()
|
||||
{
|
||||
using var source = new TrackingReadStream(new byte[100]);
|
||||
using var destination = new MemoryStream();
|
||||
using var writer = new GZipWriter(destination, new GZipWriterOptions { BufferSize = 19 });
|
||||
|
||||
writer.Write("buffer-size.txt", source, DateTime.Now);
|
||||
|
||||
Assert.Equal(19, source.CopyBufferSize);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TarWriter_Uses_WriterOptions_BufferSize()
|
||||
{
|
||||
using var source = new TrackingReadStream(new byte[100]);
|
||||
using var destination = new MemoryStream();
|
||||
using var writer = new TarWriter(
|
||||
destination,
|
||||
new TarWriterOptions(CompressionType.None, true) { BufferSize = 0 }
|
||||
);
|
||||
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() =>
|
||||
writer.Write("buffer-size.txt", source, DateTime.Now)
|
||||
);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReaderOptions_Fluent_Methods_Modify_Correctly()
|
||||
{
|
||||
@@ -375,12 +448,6 @@ public class OptionsUsabilityTests : TestBase
|
||||
{
|
||||
public int? CopyBufferSize { get; private set; }
|
||||
|
||||
public override void CopyTo(Stream destination, int bufferSize)
|
||||
{
|
||||
CopyBufferSize = bufferSize;
|
||||
base.CopyTo(destination, bufferSize);
|
||||
}
|
||||
|
||||
public override Task CopyToAsync(
|
||||
Stream destination,
|
||||
int bufferSize,
|
||||
@@ -477,12 +544,6 @@ public class OptionsUsabilityTests : TestBase
|
||||
Action<int> copyBufferSize
|
||||
) : EntryStream(reader, stream)
|
||||
{
|
||||
public override void CopyTo(Stream destination, int bufferSize)
|
||||
{
|
||||
copyBufferSize(bufferSize);
|
||||
base.CopyTo(destination, bufferSize);
|
||||
}
|
||||
|
||||
public override Task CopyToAsync(
|
||||
Stream destination,
|
||||
int bufferSize,
|
||||
|
||||
@@ -579,7 +579,7 @@ public class UtilityTests
|
||||
using var source = new MemoryStream(sourceData);
|
||||
using var destination = new MemoryStream();
|
||||
|
||||
var transferred = source.TransferTo(destination, 5);
|
||||
var transferred = source.TransferTo(destination, 5, null);
|
||||
|
||||
Assert.Equal(5, transferred);
|
||||
Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, destination.ToArray());
|
||||
@@ -592,7 +592,7 @@ public class UtilityTests
|
||||
using var source = new MemoryStream(sourceData);
|
||||
using var destination = new MemoryStream();
|
||||
|
||||
var transferred = source.TransferTo(destination, 100);
|
||||
var transferred = source.TransferTo(destination, 100, null);
|
||||
|
||||
Assert.Equal(3, transferred);
|
||||
Assert.Equal(sourceData, destination.ToArray());
|
||||
@@ -604,7 +604,7 @@ public class UtilityTests
|
||||
using var source = new MemoryStream();
|
||||
using var destination = new MemoryStream();
|
||||
|
||||
var transferred = source.TransferTo(destination, 100);
|
||||
var transferred = source.TransferTo(destination, 100, null);
|
||||
|
||||
Assert.Equal(0, transferred);
|
||||
Assert.Empty(destination.ToArray());
|
||||
|
||||
Reference in New Issue
Block a user