diff --git a/docs/API.md b/docs/API.md index 0fbcb27f..19edd6ed 100644 --- a/docs/API.md +++ b/docs/API.md @@ -92,7 +92,8 @@ using (var archive = GZipArchive.CreateArchive()) // With fluent options (preferred) var options = WriterOptions.ForZip() .WithCompressionLevel(9) - .WithLeaveStreamOpen(false); + .WithLeaveStreamOpen(false) + .WithBufferSize(131072); using (var archive = ZipArchive.CreateArchive()) { archive.SaveTo("output.zip", options); @@ -102,10 +103,13 @@ using (var archive = ZipArchive.CreateArchive()) var options2 = new WriterOptions(CompressionType.Deflate) { CompressionLevel = 9, - LeaveStreamOpen = false + LeaveStreamOpen = false, + BufferSize = 131072 }; ``` +`WriterOptions.BufferSize` controls stream copy buffers used while writing archive entries. If it is not set, SharpCompress falls back to `Constants.BufferSize`. + --- ## Archive API Methods diff --git a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs index 0f5034c1..0de4b21c 100644 --- a/src/SharpCompress/Archives/IArchiveEntryExtensions.cs +++ b/src/SharpCompress/Archives/IArchiveEntryExtensions.cs @@ -17,7 +17,8 @@ public static class IArchiveEntryExtensions /// /// The stream to write the entry content to. /// Optional progress reporter for tracking extraction progress. - public void WriteTo(Stream streamToWriteTo, IProgress? progress = null) => archiveEntry.WriteTo(streamToWriteTo, null, progress); + public void WriteTo(Stream streamToWriteTo, IProgress? progress = null) => + archiveEntry.WriteTo(streamToWriteTo, null, progress); private void WriteTo( Stream streamToWriteTo, diff --git a/src/SharpCompress/Common/Constants.cs b/src/SharpCompress/Common/Constants.cs index edd81e08..44909ef9 100644 --- a/src/SharpCompress/Common/Constants.cs +++ b/src/SharpCompress/Common/Constants.cs @@ -8,6 +8,7 @@ public static class Constants /// The default buffer size for stream operations, matching .NET's Stream.CopyTo default of 81920 bytes. /// This can be modified globally at runtime. /// + // TODO: Revisit remaining non-extraction usages after extraction buffering moves to ExtractionOptions. public static int BufferSize { get; set; } = 81920; /// diff --git a/src/SharpCompress/Common/Options/IWriterOptions.cs b/src/SharpCompress/Common/Options/IWriterOptions.cs index a7a1d719..37fbb21e 100644 --- a/src/SharpCompress/Common/Options/IWriterOptions.cs +++ b/src/SharpCompress/Common/Options/IWriterOptions.cs @@ -19,6 +19,11 @@ public interface IWriterOptions : IStreamOptions, IEncodingOptions, IProgressOpt /// int CompressionLevel { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + int BufferSize { get; set; } + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom providers, such as diff --git a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs index 3c1d0408..a982982c 100644 --- a/src/SharpCompress/Readers/IAsyncReaderExtensions.cs +++ b/src/SharpCompress/Readers/IAsyncReaderExtensions.cs @@ -1,7 +1,9 @@ +using System; using System.IO; using System.Threading; using System.Threading.Tasks; using SharpCompress.Common; +using SharpCompress.IO; namespace SharpCompress.Readers; @@ -42,7 +44,8 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) + .ConfigureAwait(false); }, cancellationToken ) @@ -77,7 +80,8 @@ public static class IAsyncReaderExtensions async (x, fm, ct) => { using var fs = File.Open(x, fm); - await reader.WriteEntryToAsync(fs, ct).ConfigureAwait(false); + await CopyEntryToAsync(reader, fs, options?.BufferSize, ct) + .ConfigureAwait(false); }, cancellationToken ) @@ -92,4 +96,58 @@ public static class IAsyncReaderExtensions .WriteEntryToAsync(destinationFileInfo.FullName, options, cancellationToken) .ConfigureAwait(false); } + + private static async ValueTask CopyEntryToAsync( + IAsyncReader reader, + Stream writableStream, + int? bufferSize, + CancellationToken cancellationToken + ) + { +#if LEGACY_DOTNET + using var entryStream = await reader + .OpenEntryStreamAsync(cancellationToken) + .ConfigureAwait(false); +#else + await using var entryStream = await reader + .OpenEntryStreamAsync(cancellationToken) + .ConfigureAwait(false); +#endif + var sourceStream = WrapWithProgress(entryStream, reader.Entry); + await sourceStream + .CopyToAsync(writableStream, bufferSize ?? Constants.BufferSize, cancellationToken) + .ConfigureAwait(false); + } + + private static Stream WrapWithProgress(Stream source, IEntry entry) + { + var progress = entry.Options.Progress; + if (progress is null) + { + return source; + } + + var entryPath = entry.Key ?? string.Empty; + var totalBytes = GetEntrySizeSafe(entry); + return new ProgressReportingStream( + source, + progress, + entryPath, + totalBytes, + leaveOpen: true + ); + } + + private static long? GetEntrySizeSafe(IEntry entry) + { + try + { + var size = entry.Size; + return size >= 0 ? size : null; + } + catch (NotImplementedException) + { + return null; + } + } } diff --git a/src/SharpCompress/Utility.Async.cs b/src/SharpCompress/Utility.Async.cs index 1c11ccc6..df323d69 100644 --- a/src/SharpCompress/Utility.Async.cs +++ b/src/SharpCompress/Utility.Async.cs @@ -65,13 +65,14 @@ internal static partial class Utility public async ValueTask TransferToAsync( Stream destination, long maxLength, + int? bufferSize = null, CancellationToken cancellationToken = default ) { // Use ReadOnlySubStream to limit reading and leverage framework's CopyToAsync using var limitedStream = new IO.ReadOnlySubStream(source, maxLength); await limitedStream - .CopyToAsync(destination, Constants.BufferSize, cancellationToken) + .CopyToAsync(destination, bufferSize ?? Constants.BufferSize, cancellationToken) .ConfigureAwait(false); return limitedStream.Position; } diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index 017bc965..c03c3643 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -150,11 +150,11 @@ internal static partial class Utility extension(Stream source) { - public long TransferTo(Stream destination, long maxLength) + public long TransferTo(Stream destination, long maxLength, int? bufferSize) { // Use ReadOnlySubStream to limit reading and leverage framework's CopyTo using var limitedStream = new IO.ReadOnlySubStream(source, maxLength); - limitedStream.CopyTo(destination, Constants.BufferSize); + limitedStream.CopyTo(destination, bufferSize ?? Constants.BufferSize); return limitedStream.Position; } diff --git a/src/SharpCompress/Writers/GZip/GZipWriter.cs b/src/SharpCompress/Writers/GZip/GZipWriter.cs index 29c9425c..3c62ff2d 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriter.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriter.cs @@ -86,7 +86,7 @@ public sealed partial class GZipWriter : AbstractWriter } var progressStream = WrapWithProgress(source, filename); - progressStream.CopyTo(OutputStream.NotNull(), Constants.BufferSize); + progressStream.CopyTo(OutputStream.NotNull(), WriterOptions.BufferSize); _wroteToStream = true; } diff --git a/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs b/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs index 42da70e2..ca1b6a9d 100644 --- a/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs +++ b/src/SharpCompress/Writers/GZip/GZipWriterOptions.cs @@ -68,6 +68,11 @@ public sealed record GZipWriterOptions : IWriterOptions /// public IProgress? Progress { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + public int BufferSize { get; set; } = Constants.BufferSize; + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom implementations, such as @@ -115,6 +120,7 @@ public sealed record GZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } @@ -128,6 +134,7 @@ public sealed record GZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } } diff --git a/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs b/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs index 86fa09ff..6ed26a42 100644 --- a/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs +++ b/src/SharpCompress/Writers/SevenZip/SevenZipWriterOptions.cs @@ -57,6 +57,11 @@ public sealed record SevenZipWriterOptions : IWriterOptions /// public IProgress? Progress { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + public int BufferSize { get; set; } = Constants.BufferSize; + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom implementations. @@ -103,6 +108,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } @@ -117,6 +123,7 @@ public sealed record SevenZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } diff --git a/src/SharpCompress/Writers/Tar/TarWriter.Async.cs b/src/SharpCompress/Writers/Tar/TarWriter.Async.cs index 7f4b3886..9244608f 100644 --- a/src/SharpCompress/Writers/Tar/TarWriter.Async.cs +++ b/src/SharpCompress/Writers/Tar/TarWriter.Async.cs @@ -104,7 +104,12 @@ public partial class TarWriter await header.WriteAsync(OutputStream.NotNull(), cancellationToken).ConfigureAwait(false); var progressStream = WrapWithProgress(source, filename); var written = await progressStream - .TransferToAsync(OutputStream.NotNull(), realSize, cancellationToken) + .TransferToAsync( + OutputStream.NotNull(), + realSize, + WriterOptions.BufferSize, + cancellationToken + ) .ConfigureAwait(false); await PadTo512Async(written, cancellationToken).ConfigureAwait(false); } diff --git a/src/SharpCompress/Writers/Tar/TarWriter.cs b/src/SharpCompress/Writers/Tar/TarWriter.cs index 4ed20335..8136cc58 100644 --- a/src/SharpCompress/Writers/Tar/TarWriter.cs +++ b/src/SharpCompress/Writers/Tar/TarWriter.cs @@ -133,7 +133,11 @@ public partial class TarWriter : AbstractWriter header.Size = realSize; header.Write(OutputStream.NotNull()); var progressStream = WrapWithProgress(source, filename); - size = progressStream.TransferTo(OutputStream.NotNull(), realSize); + size = progressStream.TransferTo( + OutputStream.NotNull(), + realSize, + WriterOptions.BufferSize + ); PadTo512(size.Value); } diff --git a/src/SharpCompress/Writers/Tar/TarWriterOptions.cs b/src/SharpCompress/Writers/Tar/TarWriterOptions.cs index 004d4643..d7aaf22b 100755 --- a/src/SharpCompress/Writers/Tar/TarWriterOptions.cs +++ b/src/SharpCompress/Writers/Tar/TarWriterOptions.cs @@ -44,6 +44,11 @@ public sealed record TarWriterOptions : IWriterOptions /// public IProgress? Progress { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + public int BufferSize { get; set; } = Constants.BufferSize; + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom implementations. @@ -104,6 +109,7 @@ public sealed record TarWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } @@ -118,6 +124,7 @@ public sealed record TarWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } diff --git a/src/SharpCompress/Writers/WriterOptions.cs b/src/SharpCompress/Writers/WriterOptions.cs index e4dd4ac0..d70ea6aa 100644 --- a/src/SharpCompress/Writers/WriterOptions.cs +++ b/src/SharpCompress/Writers/WriterOptions.cs @@ -56,6 +56,11 @@ public sealed record WriterOptions : IWriterOptions /// public IProgress? Progress { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + public int BufferSize { get; set; } = Constants.BufferSize; + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom implementations, such as diff --git a/src/SharpCompress/Writers/WriterOptionsExtensions.cs b/src/SharpCompress/Writers/WriterOptionsExtensions.cs index 964d9187..2aca567a 100644 --- a/src/SharpCompress/Writers/WriterOptionsExtensions.cs +++ b/src/SharpCompress/Writers/WriterOptionsExtensions.cs @@ -53,6 +53,15 @@ public static class WriterOptionsExtensions ), }; + /// + /// Creates a copy with the specified buffer size. + /// + public static WriterOptions WithBufferSize(this WriterOptions options, int bufferSize) => + options with + { + BufferSize = bufferSize, + }; + /// /// Creates a copy with the specified compression level. /// diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index 51695341..128cb9d7 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -16,7 +16,6 @@ using SharpCompress.Compressors.PPMd; using SharpCompress.Compressors.ZStandard; using SharpCompress.IO; using SharpCompress.Providers; -using Constants = SharpCompress.Common.Constants; namespace SharpCompress.Writers.Zip; @@ -89,7 +88,7 @@ public partial class ZipWriter : AbstractWriter { using var output = WriteToStream(entryPath, zipWriterEntryOptions); var progressStream = WrapWithProgress(source, entryPath); - progressStream.CopyTo(output, Constants.BufferSize); + progressStream.CopyTo(output, WriterOptions.BufferSize); } public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options) diff --git a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs index e3bf2834..73c92ee6 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs @@ -61,6 +61,11 @@ public sealed record ZipWriterOptions : IWriterOptions /// public IProgress? Progress { get; set; } + /// + /// Buffer size for writer stream copy operations. + /// + public int BufferSize { get; set; } = Constants.BufferSize; + /// /// Registry of compression providers. /// Defaults to but can be replaced with custom implementations. @@ -128,6 +133,7 @@ public sealed record ZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } @@ -141,6 +147,7 @@ public sealed record ZipWriterOptions : IWriterOptions LeaveStreamOpen = options.LeaveStreamOpen; ArchiveEncoding = options.ArchiveEncoding; Progress = options.Progress; + BufferSize = options.BufferSize; Providers = options.Providers; } diff --git a/tests/SharpCompress.Test/OptionsUsabilityTests.cs b/tests/SharpCompress.Test/OptionsUsabilityTests.cs index b5a997ab..85aebb73 100644 --- a/tests/SharpCompress.Test/OptionsUsabilityTests.cs +++ b/tests/SharpCompress.Test/OptionsUsabilityTests.cs @@ -11,6 +11,7 @@ using SharpCompress.Readers; using SharpCompress.Test.Mocks; using SharpCompress.Writers; using SharpCompress.Writers.GZip; +using SharpCompress.Writers.Tar; using SharpCompress.Writers.Zip; using Xunit; @@ -133,11 +134,16 @@ public class OptionsUsabilityTests : TestBase [Fact] public void WriterOptions_Fluent_Methods_Modify_Correctly() { - var options = WriterOptions.ForZip().WithLeaveStreamOpen(false).WithCompressionLevel(9); + var options = WriterOptions + .ForZip() + .WithLeaveStreamOpen(false) + .WithCompressionLevel(9) + .WithBufferSize(65536); Assert.Equal(CompressionType.Deflate, options.CompressionType); Assert.Equal(9, options.CompressionLevel); Assert.False(options.LeaveStreamOpen); + Assert.Equal(65536, options.BufferSize); } [Fact] @@ -161,6 +167,73 @@ public class OptionsUsabilityTests : TestBase Assert.Equal(factoryApproach.LeaveStreamOpen, constructorApproach.LeaveStreamOpen); } + [Fact] + public void WriterOptions_Default_BufferSize_Uses_Constants_BufferSize() + { + Assert.Equal(Constants.BufferSize, WriterOptions.ForZip().BufferSize); + Assert.Equal( + Constants.BufferSize, + new ZipWriterOptions(CompressionType.Deflate).BufferSize + ); + Assert.Equal( + Constants.BufferSize, + new TarWriterOptions(CompressionType.None, true).BufferSize + ); + Assert.Equal(Constants.BufferSize, new GZipWriterOptions().BufferSize); + } + + [Fact] + public void Format_WriterOptions_Copy_BufferSize() + { + var options = WriterOptions.ForZip().WithBufferSize(12345); + + Assert.Equal(12345, new ZipWriterOptions(options).BufferSize); + Assert.Equal(12345, new TarWriterOptions(options).BufferSize); + Assert.Equal(12345, new GZipWriterOptions(options).BufferSize); + } + + [Fact] + public void ZipWriter_Uses_WriterOptions_BufferSize() + { + using var source = new TrackingReadStream(new byte[100]); + using var destination = new MemoryStream(); + using var writer = new ZipWriter( + destination, + new ZipWriterOptions(CompressionType.None) { BufferSize = 17 } + ); + + writer.Write("buffer-size.txt", source, DateTime.Now); + + Assert.Equal(17, source.CopyBufferSize); + } + + [Fact] + public void GZipWriter_Uses_WriterOptions_BufferSize() + { + using var source = new TrackingReadStream(new byte[100]); + using var destination = new MemoryStream(); + using var writer = new GZipWriter(destination, new GZipWriterOptions { BufferSize = 19 }); + + writer.Write("buffer-size.txt", source, DateTime.Now); + + Assert.Equal(19, source.CopyBufferSize); + } + + [Fact] + public void TarWriter_Uses_WriterOptions_BufferSize() + { + using var source = new TrackingReadStream(new byte[100]); + using var destination = new MemoryStream(); + using var writer = new TarWriter( + destination, + new TarWriterOptions(CompressionType.None, true) { BufferSize = 0 } + ); + + Assert.Throws(() => + writer.Write("buffer-size.txt", source, DateTime.Now) + ); + } + [Fact] public void ReaderOptions_Fluent_Methods_Modify_Correctly() { @@ -375,12 +448,6 @@ public class OptionsUsabilityTests : TestBase { public int? CopyBufferSize { get; private set; } - public override void CopyTo(Stream destination, int bufferSize) - { - CopyBufferSize = bufferSize; - base.CopyTo(destination, bufferSize); - } - public override Task CopyToAsync( Stream destination, int bufferSize, @@ -477,12 +544,6 @@ public class OptionsUsabilityTests : TestBase Action copyBufferSize ) : EntryStream(reader, stream) { - public override void CopyTo(Stream destination, int bufferSize) - { - copyBufferSize(bufferSize); - base.CopyTo(destination, bufferSize); - } - public override Task CopyToAsync( Stream destination, int bufferSize, diff --git a/tests/SharpCompress.Test/UtilityTests.cs b/tests/SharpCompress.Test/UtilityTests.cs index 20d46e6b..4dbe2f83 100644 --- a/tests/SharpCompress.Test/UtilityTests.cs +++ b/tests/SharpCompress.Test/UtilityTests.cs @@ -579,7 +579,7 @@ public class UtilityTests using var source = new MemoryStream(sourceData); using var destination = new MemoryStream(); - var transferred = source.TransferTo(destination, 5); + var transferred = source.TransferTo(destination, 5, null); Assert.Equal(5, transferred); Assert.Equal(new byte[] { 1, 2, 3, 4, 5 }, destination.ToArray()); @@ -592,7 +592,7 @@ public class UtilityTests using var source = new MemoryStream(sourceData); using var destination = new MemoryStream(); - var transferred = source.TransferTo(destination, 100); + var transferred = source.TransferTo(destination, 100, null); Assert.Equal(3, transferred); Assert.Equal(sourceData, destination.ToArray()); @@ -604,7 +604,7 @@ public class UtilityTests using var source = new MemoryStream(); using var destination = new MemoryStream(); - var transferred = source.TransferTo(destination, 100); + var transferred = source.TransferTo(destination, 100, null); Assert.Equal(0, transferred); Assert.Empty(destination.ToArray());