starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-11-15 17:02:03 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Release TAR probe buffers and avoid buffering seekable inputs

Browse Source
This commit is contained in:
Adam Hathcock
2026-10-15 14:05:19 +01:00
parent f5f8558de6
commit c063e45d1b
7 changed files with 533 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs
View File

@@ -77,6 +77,9 @@ internal partial class ZlibBaseStream : Stream, IStreamStack
internal int Crc32 => crc?.Crc32Result ?? 0;
// Format detection reads only a prefix, so no complete GZIP trailer is available.
internal bool IsFormatProbe { get; set; }
public ZlibBaseStream(
Stream stream,
CompressionMode compressionMode,
@@ -271,7 +274,7 @@ internal partial class ZlibBaseStream : Stream, IStreamStack
// workitem 7159
else if (_streamMode == StreamMode.Reader)
{
if (_flavor == ZlibStreamFlavor.GZIP)
if (_flavor == ZlibStreamFlavor.GZIP && !IsFormatProbe)
{
if (!_wantCompress)
{
@@ -385,7 +388,7 @@ internal partial class ZlibBaseStream : Stream, IStreamStack
// workitem 7159
else if (_streamMode == StreamMode.Reader)
{
if (_flavor == ZlibStreamFlavor.GZIP)
if (_flavor == ZlibStreamFlavor.GZIP && !IsFormatProbe)
{
if (!_wantCompress)
{

234
src/SharpCompress/Factories/TarFactory.cs
View File

@@ -7,6 +7,7 @@ using SharpCompress.Archives;
using SharpCompress.Archives.Tar;
using SharpCompress.Common;
using SharpCompress.Common.Options;
using SharpCompress.Compressors.Deflate;
using SharpCompress.IO;
using SharpCompress.Providers;
using SharpCompress.Readers;
@@ -65,6 +66,15 @@ public class TarFactory
#endregion
internal static Stream PrepareProbeStream(Stream stream)
{
if (stream is GZipStream gzip)
{
gzip.BaseStream.IsFormatProbe = true;
}
return stream;
}
private static Stream CreateProbeDecompressionStream(
Stream stream,
CompressionType compressionType,
@@ -80,14 +90,20 @@ public class TarFactory
if (compressionType == CompressionType.GZip && readerOptions is not null)
{
return providers.CreateDecompressStream(
compressionType,
nonDisposingStream,
CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(readerOptions)
return PrepareProbeStream(
providers.CreateDecompressStream(
compressionType,
nonDisposingStream,
CompressionContext
.FromStream(nonDisposingStream)
.WithReaderOptions(readerOptions)
)
);
}
return providers.CreateDecompressStream(compressionType, nonDisposingStream);
return PrepareProbeStream(
providers.CreateDecompressStream(compressionType, nonDisposingStream)
);
}
private static async ValueTask<Stream> CreateProbeDecompressionStreamAsync(
@@ -106,21 +122,25 @@ public class TarFactory
if (compressionType == CompressionType.GZip && readerOptions is not null)
{
return await providers
.CreateDecompressStreamAsync(
compressionType,
nonDisposingStream,
CompressionContext
.FromStream(nonDisposingStream)
.WithReaderOptions(readerOptions),
cancellationToken
)
.ConfigureAwait(false);
return PrepareProbeStream(
await providers
.CreateDecompressStreamAsync(
compressionType,
nonDisposingStream,
CompressionContext
.FromStream(nonDisposingStream)
.WithReaderOptions(readerOptions),
cancellationToken
)
.ConfigureAwait(false)
);
}
return await providers
.CreateDecompressStreamAsync(compressionType, nonDisposingStream, cancellationToken)
.ConfigureAwait(false);
return PrepareProbeStream(
await providers
.CreateDecompressStreamAsync(compressionType, nonDisposingStream, cancellationToken)
.ConfigureAwait(false)
);
}
public static CompressionType GetCompressionType(
@@ -135,7 +155,7 @@ public class TarFactory
if (wrapper.IsMatch(stream))
{
stream.Seek(0, SeekOrigin.Begin);
var decompressedStream = CreateProbeDecompressionStream(
using var decompressedStream = CreateProbeDecompressionStream(
stream,
wrapper.CompressionType,
readerOptions
@@ -162,7 +182,11 @@ public class TarFactory
if (await wrapper.IsMatchAsync(stream, cancellationToken).ConfigureAwait(false))
{
stream.Seek(0, SeekOrigin.Begin);
var decompressedStream = await CreateProbeDecompressionStreamAsync(
#if LEGACY_DOTNET && !NETSTANDARD2_1
using var decompressedStream = await CreateProbeDecompressionStreamAsync(
#else
await using var decompressedStream = await CreateProbeDecompressionStreamAsync(
#endif
stream,
wrapper.CompressionType,
readerOptions,
@@ -338,32 +362,96 @@ public class TarFactory
return new SeekableSharpCompressStream(stream, leaveStreamOpen: false);
}
private static SharpCompressStream CreateReaderProbeStream(Stream stream, ReaderOptions options)
{
var borrowedStream = SharpCompressStream.CreateNonDisposing(stream);
var bufferSize = Math.Max(
options.RewindableBufferSize ?? 0,
TarWrapper.MaximumRewindBufferSize
);
var probe = CreateNestedRecordingStream(borrowedStream);
probe.StartRecording(bufferSize);
return probe;
}
internal static async ValueTask<SharpCompressStream> BufferLZipHeaderAsync(
SharpCompressStream stream,
CancellationToken cancellationToken
)
{
if (stream is not SeekableSharpCompressStream)
{
return stream;
}
// LZip initializes both its six-byte header and five-byte range coder
// synchronously. Cache those bytes with async I/O before creating it.
var buffered = new SharpCompressStream(stream);
buffered.StartRecording();
try
{
var header = new byte[11];
await buffered
.ReadFullyAsync(header, 0, header.Length, cancellationToken)
.ConfigureAwait(false);
buffered.Rewind();
return buffered;
}
catch
{
#if LEGACY_DOTNET && !NETSTANDARD2_1
buffered.Dispose();
#else
await buffered.DisposeAsync().ConfigureAwait(false);
#endif
throw;
}
}
/// <inheritdoc/>
public IReader OpenReader(Stream stream, ReaderOptions? options)
{
options ??= ReaderOptions.ForExternalStream;
var sharpCompressStream = CreateNestedRecordingStream(stream);
sharpCompressStream.StartRecording(TarWrapper.MaximumRewindBufferSize);
foreach (var wrapper in TarWrapper.Wrappers)
var sharpCompressStream = CreateReaderProbeStream(stream, options);
var transferred = false;
try
{
sharpCompressStream.Rewind();
if (wrapper.IsMatch(sharpCompressStream))
foreach (var wrapper in TarWrapper.Wrappers)
{
sharpCompressStream.Rewind();
var decompressedStream = CreateProbeDecompressionStream(
sharpCompressStream,
wrapper.CompressionType,
options
);
if (TarArchive.IsTarFile(decompressedStream))
if (wrapper.IsMatch(sharpCompressStream))
{
sharpCompressStream.Rewind();
sharpCompressStream.StopRecording();
return new TarReader(sharpCompressStream, options, wrapper.CompressionType);
using var decompressedStream = CreateProbeDecompressionStream(
sharpCompressStream,
wrapper.CompressionType,
options
);
if (TarArchive.IsTarFile(decompressedStream))
{
sharpCompressStream.StopRecording();
var reader = new TarReader(
sharpCompressStream,
options with
{
LeaveStreamOpen = false,
},
wrapper.CompressionType,
options.LeaveStreamOpen ? null : stream
);
transferred = true;
return reader;
}
}
}
throw new InvalidFormatException("Not a tar file.");
}
finally
{
if (!transferred)
{
sharpCompressStream.Dispose();
}
}
throw new InvalidFormatException("Not a tar file.");
}
/// <inheritdoc/>
@@ -375,39 +463,73 @@ public class TarFactory
{
cancellationToken.ThrowIfCancellationRequested();
options ??= ReaderOptions.ForExternalStream;
var sharpCompressStream = new SharpCompressStream(stream);
sharpCompressStream.StartRecording(TarWrapper.MaximumRewindBufferSize);
foreach (var wrapper in TarWrapper.Wrappers)
var sharpCompressStream = CreateReaderProbeStream(stream, options);
var transferred = false;
try
{
sharpCompressStream.Rewind();
if (
await wrapper
.IsMatchAsync(sharpCompressStream, cancellationToken)
.ConfigureAwait(false)
)
foreach (var wrapper in TarWrapper.Wrappers)
{
sharpCompressStream.Rewind();
var decompressedStream = await CreateProbeDecompressionStreamAsync(
sharpCompressStream,
wrapper.CompressionType,
options,
cancellationToken
)
.ConfigureAwait(false);
if (
await TarArchive
.IsTarFileAsync(decompressedStream, cancellationToken)
await wrapper
.IsMatchAsync(sharpCompressStream, cancellationToken)
.ConfigureAwait(false)
)
{
sharpCompressStream.Rewind();
sharpCompressStream.StopRecording();
return new TarReader(sharpCompressStream, options, wrapper.CompressionType);
if (wrapper.CompressionType == CompressionType.LZip)
{
sharpCompressStream = await BufferLZipHeaderAsync(
sharpCompressStream,
cancellationToken
)
.ConfigureAwait(false);
}
#if LEGACY_DOTNET && !NETSTANDARD2_1
using var decompressedStream = await CreateProbeDecompressionStreamAsync(
#else
await using var decompressedStream = await CreateProbeDecompressionStreamAsync(
#endif
sharpCompressStream,
wrapper.CompressionType,
options,
cancellationToken
)
.ConfigureAwait(false);
if (
await TarArchive
.IsTarFileAsync(decompressedStream, cancellationToken)
.ConfigureAwait(false)
)
{
sharpCompressStream.StopRecording();
var reader = new TarReader(
sharpCompressStream,
options with
{
LeaveStreamOpen = false,
},
wrapper.CompressionType,
options.LeaveStreamOpen ? null : stream
);
transferred = true;
return reader;
}
}
}
throw new InvalidFormatException("Not a tar file.");
}
finally
{
if (!transferred)
{
#if LEGACY_DOTNET && !NETSTANDARD2_1
sharpCompressStream.Dispose();
#else
await sharpCompressStream.DisposeAsync().ConfigureAwait(false);
#endif
}
}
throw new InvalidFormatException("Not a tar file.");
}
#endregion

7
src/SharpCompress/Readers/ReaderFactory.Async.cs
View File

@@ -59,6 +59,13 @@ public static partial class ReaderFactory
stream.RequireReadable();
options ??= ReaderOptions.ForExternalStream;
if (options.LeaveStreamOpen)
{
// Internal wrappers must still be disposed when the caller retains its stream.
stream = SharpCompressStream.CreateNonDisposing(stream);
options = options with { LeaveStreamOpen = false };
}
var sharpCompressStream = SharpCompressStream.Create(
stream,
bufferSize: options.RewindableBufferSize

7
src/SharpCompress/Readers/ReaderFactory.cs
View File

@@ -32,6 +32,13 @@ public static partial class ReaderFactory
stream.RequireReadable();
options ??= ReaderOptions.ForExternalStream;
if (options.LeaveStreamOpen)
{
// Protect the caller's stream, not the internal wrappers and their pooled buffers.
stream = SharpCompressStream.CreateNonDisposing(stream);
options = options with { LeaveStreamOpen = false };
}
var sharpCompressStream = SharpCompressStream.Create(
stream,
bufferSize: options.RewindableBufferSize

57
src/SharpCompress/Readers/Tar/TarReader.Factory.cs
View File

@@ -30,10 +30,12 @@ public partial class TarReader
if (compressionType == CompressionType.GZip)
{
return providers.CreateDecompressStream(
compressionType,
nonDisposingStream,
CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options)
return TarFactory.PrepareProbeStream(
providers.CreateDecompressStream(
compressionType,
nonDisposingStream,
CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options)
)
);
}
@@ -56,14 +58,18 @@ public partial class TarReader
if (compressionType == CompressionType.GZip)
{
return await providers
.CreateDecompressStreamAsync(
compressionType,
nonDisposingStream,
CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options),
cancellationToken
)
.ConfigureAwait(false);
return TarFactory.PrepareProbeStream(
await providers
.CreateDecompressStreamAsync(
compressionType,
nonDisposingStream,
CompressionContext
.FromStream(nonDisposingStream)
.WithReaderOptions(options),
cancellationToken
)
.ConfigureAwait(false)
);
}
return await providers
@@ -90,6 +96,11 @@ public partial class TarReader
cancellationToken.ThrowIfCancellationRequested();
stream.NotNull(nameof(stream));
readerOptions ??= ReaderOptions.ForExternalStream;
if (readerOptions.LeaveStreamOpen)
{
stream = SharpCompressStream.CreateNonDisposing(stream);
readerOptions = readerOptions with { LeaveStreamOpen = false };
}
var sharpCompressStream = SharpCompressStream.Create(
stream,
bufferSize: Math.Max(
@@ -97,6 +108,7 @@ public partial class TarReader
TarWrapper.MaximumRewindBufferSize
)
);
sharpCompressStream.StartRecording();
long pos = sharpCompressStream.Position;
foreach (var wrapper in TarWrapper.Wrappers)
{
@@ -111,7 +123,18 @@ public partial class TarReader
}
sharpCompressStream.Position = pos;
var testStream = await CreateProbeDecompressionStreamAsync(
if (wrapper.CompressionType == CompressionType.LZip)
{
sharpCompressStream = await TarFactory
.BufferLZipHeaderAsync(sharpCompressStream, cancellationToken)
.ConfigureAwait(false);
pos = sharpCompressStream.Position;
}
#if LEGACY_DOTNET && !NETSTANDARD2_1
using var testStream = await CreateProbeDecompressionStreamAsync(
#else
await using var testStream = await CreateProbeDecompressionStreamAsync(
#endif
sharpCompressStream,
wrapper.CompressionType,
readerOptions.Providers,
@@ -172,6 +195,11 @@ public partial class TarReader
{
stream.RequireReadable();
readerOptions ??= ReaderOptions.ForExternalStream;
if (readerOptions.LeaveStreamOpen)
{
stream = SharpCompressStream.CreateNonDisposing(stream);
readerOptions = readerOptions with { LeaveStreamOpen = false };
}
var sharpCompressStream = SharpCompressStream.Create(
stream,
bufferSize: Math.Max(
@@ -179,6 +207,7 @@ public partial class TarReader
TarWrapper.MaximumRewindBufferSize
)
);
sharpCompressStream.StartRecording();
long pos = sharpCompressStream.Position;
foreach (var wrapper in TarWrapper.Wrappers)
{
@@ -189,7 +218,7 @@ public partial class TarReader
}
sharpCompressStream.Position = pos;
var testStream = CreateProbeDecompressionStream(
using var testStream = CreateProbeDecompressionStream(
sharpCompressStream,
wrapper.CompressionType,
readerOptions.Providers,

51
src/SharpCompress/Readers/Tar/TarReader.cs
View File

@@ -20,16 +20,65 @@ namespace SharpCompress.Readers.Tar;
public partial class TarReader : AbstractReader<TarEntry, TarVolume>
{
private readonly CompressionType compressionType;
private readonly Stream? ownedSource;
private bool disposed;
internal TarReader(Stream stream, ReaderOptions options, CompressionType compressionType)
internal TarReader(
Stream stream,
ReaderOptions options,
CompressionType compressionType,
Stream? ownedSource = null
)
: base(options, ArchiveType.Tar)
{
this.compressionType = compressionType;
this.ownedSource = ownedSource;
Volume = new TarVolume(stream, options);
}
public override TarVolume Volume { get; }
public override void Dispose()
{
if (disposed)
{
return;
}
disposed = true;
try
{
base.Dispose();
}
finally
{
ownedSource?.Dispose();
}
}
public override async ValueTask DisposeAsync()
{
if (disposed)
{
return;
}
disposed = true;
try
{
await base.DisposeAsync().ConfigureAwait(false);
}
finally
{
#if LEGACY_DOTNET && !NETSTANDARD2_1
ownedSource?.Dispose();
#else
if (ownedSource is not null)
{
await ownedSource.DisposeAsync().ConfigureAwait(false);
}
#endif
}
}
protected override Stream RequestInitialStream()
{
var stream = base.RequestInitialStream();

243
tests/SharpCompress.Test/Streams/TarLifetimeTests.cs Normal file
View File

@@ -0,0 +1,243 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.Deflate;
using SharpCompress.Factories;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.Tar;
using SharpCompress.Test.Mocks;
using SharpCompress.Writers;
using Xunit;
namespace SharpCompress.Test.Streams;
public class TarLifetimeTests : TestBase
{
public static IEnumerable<object[]> ReaderCases()
{
foreach (
var filename in new[]
{
"Tar.tar",
"Tar.tar.gz",
"Tar.tar.bz2",
"Tar.tar.zst",
"Tar.tar.lz",
"Tar.tar.xz",
"Tar.tar.Z",
}
)
{
foreach (var seekable in new[] { false, true })
{
foreach (var leaveOpen in new[] { false, true })
{
foreach (var async in new[] { false, true })
{
foreach (var factory in new[] { 0, 1, 2 })
{
yield return [filename, seekable, leaveOpen, async, factory];
}
}
}
}
}
}
[Theory]
[MemberData(nameof(ReaderCases))]
public async Task Reader_Dispose_Releases_Internal_Buffers(
string filename,
bool seekable,
bool leaveOpen,
bool async,
int factory
)
{
using var file = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename));
using Stream input = seekable ? file : new ForwardOnlyStream(file);
var options = new ReaderOptions { LeaveStreamOpen = leaveOpen };
var reader = async
? (TarReader)await OpenAsyncReader(input, options, factory)
: (TarReader)OpenReader(input, options, factory);
var rings = GetRings(reader.Volume.Stream);
if (seekable && !(async && filename == "Tar.tar.lz"))
{
Assert.Empty(rings);
}
else if (!seekable)
{
Assert.Contains(rings, ring => ring.Capacity >= TarWrapper.MaximumRewindBufferSize);
}
else
{
Assert.NotEmpty(rings);
}
var entries = 0;
try
{
while (
async
? await reader.MoveToNextEntryAsync(CancellationToken.None)
: reader.MoveToNextEntry()
)
{
if (!reader.Entry.IsDirectory)
{
if (async)
{
await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None);
}
else
{
reader.WriteEntryTo(Stream.Null);
}
entries++;
}
}
}
finally
{
if (async)
{
await reader.DisposeAsync();
await reader.DisposeAsync();
}
reader.Dispose();
}
Assert.True(entries > 0);
var buffer = typeof(RingBuffer).GetField(
"_buffer",
BindingFlags.Instance | BindingFlags.NonPublic
)!;
Assert.All(rings, ring => Assert.Null(buffer.GetValue(ring)));
Assert.Equal(leaveOpen, input.CanRead);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task BZip2_Probe_Preserves_Large_NonSeekable_Block(bool async)
{
var content = new byte[850_000];
new Random(42).NextBytes(content);
using var archive = new MemoryStream();
using (
var writer = WriterFactory.OpenWriter(
archive,
ArchiveType.Tar,
new WriterOptions(CompressionType.BZip2) { LeaveStreamOpen = true }
)
)
using (var source = new MemoryStream(content))
{
writer.Write("random.bin", source);
}
Assert.True(archive.Length > 81920);
archive.Position = 0;
using var input = new ForwardOnlyStream(archive);
if (async)
{
await using var reader = await ReaderFactory.OpenAsyncReader(
input,
cancellationToken: CancellationToken.None
);
Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None));
using var output = new MemoryStream();
await reader.WriteEntryToAsync(output, CancellationToken.None);
Assert.Equal(content, output.ToArray());
}
else
{
using var reader = ReaderFactory.OpenReader(input);
Assert.True(reader.MoveToNextEntry());
using var output = new MemoryStream();
reader.WriteEntryTo(output);
Assert.Equal(content, output.ToArray());
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Failed_Tar_Probe_Does_Not_Close_Source(bool async)
{
using var source = new MemoryStream();
var content = new byte[1024];
new Random(42).NextBytes(content);
using (
var gzip = new GZipStream(
SharpCompressStream.CreateNonDisposing(source),
SharpCompress.Compressors.CompressionMode.Compress
)
)
{
gzip.Write(content, 0, content.Length);
}
source.Position = 0;
using var input = new ForwardOnlyStream(source);
var factory = new TarFactory();
if (async)
{
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
await factory.OpenAsyncReader(
input,
ReaderOptions.ForExternalStream,
CancellationToken.None
)
);
}
else
{
Assert.Throws<InvalidFormatException>(() =>
factory.OpenReader(input, ReaderOptions.ForExternalStream)
);
}
Assert.True(input.CanRead);
}
private static IReader OpenReader(Stream input, ReaderOptions options, int factory) =>
factory switch
{
0 => ReaderFactory.OpenReader(input, options),
1 => new TarFactory().OpenReader(input, options),
_ => TarReader.OpenReader(input, options),
};
private static ValueTask<IAsyncReader> OpenAsyncReader(
Stream input,
ReaderOptions options,
int factory
) =>
factory switch
{
0 => ReaderFactory.OpenAsyncReader(input, options, CancellationToken.None),
1 => new TarFactory().OpenAsyncReader(input, options, CancellationToken.None),
_ => TarReader.OpenAsyncReader(input, options, CancellationToken.None),
};
private static List<RingBuffer> GetRings(Stream stream)
{
var rings = new List<RingBuffer>();
var field = typeof(SharpCompressStream).GetField(
"_ringBuffer",
BindingFlags.Instance | BindingFlags.NonPublic
)!;
while (stream is SharpCompressStream wrapper)
{
if (field.GetValue(wrapper) is RingBuffer ring)
{
rings.Add(ring);
}
stream = wrapper.BaseStream();
}
return rings;
}
}