diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index b574e74c..b4920369 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -77,6 +77,9 @@ internal partial class ZlibBaseStream : Stream, IStreamStack internal int Crc32 => crc?.Crc32Result ?? 0; + // Format detection reads only a prefix, so no complete GZIP trailer is available. + internal bool IsFormatProbe { get; set; } + public ZlibBaseStream( Stream stream, CompressionMode compressionMode, @@ -271,7 +274,7 @@ internal partial class ZlibBaseStream : Stream, IStreamStack // workitem 7159 else if (_streamMode == StreamMode.Reader) { - if (_flavor == ZlibStreamFlavor.GZIP) + if (_flavor == ZlibStreamFlavor.GZIP && !IsFormatProbe) { if (!_wantCompress) { @@ -385,7 +388,7 @@ internal partial class ZlibBaseStream : Stream, IStreamStack // workitem 7159 else if (_streamMode == StreamMode.Reader) { - if (_flavor == ZlibStreamFlavor.GZIP) + if (_flavor == ZlibStreamFlavor.GZIP && !IsFormatProbe) { if (!_wantCompress) { diff --git a/src/SharpCompress/Factories/TarFactory.cs b/src/SharpCompress/Factories/TarFactory.cs index 38dd7ab9..ab6b50a2 100644 --- a/src/SharpCompress/Factories/TarFactory.cs +++ b/src/SharpCompress/Factories/TarFactory.cs @@ -7,6 +7,7 @@ using SharpCompress.Archives; using SharpCompress.Archives.Tar; using SharpCompress.Common; using SharpCompress.Common.Options; +using SharpCompress.Compressors.Deflate; using SharpCompress.IO; using SharpCompress.Providers; using SharpCompress.Readers; @@ -65,6 +66,15 @@ public class TarFactory #endregion + internal static Stream PrepareProbeStream(Stream stream) + { + if (stream is GZipStream gzip) + { + gzip.BaseStream.IsFormatProbe = true; + } + return stream; + } + private static Stream CreateProbeDecompressionStream( Stream stream, CompressionType compressionType, @@ -80,14 +90,20 @@ public class TarFactory if (compressionType == CompressionType.GZip && readerOptions is not null) { - return providers.CreateDecompressStream( - compressionType, - nonDisposingStream, - CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(readerOptions) + return PrepareProbeStream( + providers.CreateDecompressStream( + compressionType, + nonDisposingStream, + CompressionContext + .FromStream(nonDisposingStream) + .WithReaderOptions(readerOptions) + ) ); } - return providers.CreateDecompressStream(compressionType, nonDisposingStream); + return PrepareProbeStream( + providers.CreateDecompressStream(compressionType, nonDisposingStream) + ); } private static async ValueTask CreateProbeDecompressionStreamAsync( @@ -106,21 +122,25 @@ public class TarFactory if (compressionType == CompressionType.GZip && readerOptions is not null) { - return await providers - .CreateDecompressStreamAsync( - compressionType, - nonDisposingStream, - CompressionContext - .FromStream(nonDisposingStream) - .WithReaderOptions(readerOptions), - cancellationToken - ) - .ConfigureAwait(false); + return PrepareProbeStream( + await providers + .CreateDecompressStreamAsync( + compressionType, + nonDisposingStream, + CompressionContext + .FromStream(nonDisposingStream) + .WithReaderOptions(readerOptions), + cancellationToken + ) + .ConfigureAwait(false) + ); } - return await providers - .CreateDecompressStreamAsync(compressionType, nonDisposingStream, cancellationToken) - .ConfigureAwait(false); + return PrepareProbeStream( + await providers + .CreateDecompressStreamAsync(compressionType, nonDisposingStream, cancellationToken) + .ConfigureAwait(false) + ); } public static CompressionType GetCompressionType( @@ -135,7 +155,7 @@ public class TarFactory if (wrapper.IsMatch(stream)) { stream.Seek(0, SeekOrigin.Begin); - var decompressedStream = CreateProbeDecompressionStream( + using var decompressedStream = CreateProbeDecompressionStream( stream, wrapper.CompressionType, readerOptions @@ -162,7 +182,11 @@ public class TarFactory if (await wrapper.IsMatchAsync(stream, cancellationToken).ConfigureAwait(false)) { stream.Seek(0, SeekOrigin.Begin); - var decompressedStream = await CreateProbeDecompressionStreamAsync( +#if LEGACY_DOTNET && !NETSTANDARD2_1 + using var decompressedStream = await CreateProbeDecompressionStreamAsync( +#else + await using var decompressedStream = await CreateProbeDecompressionStreamAsync( +#endif stream, wrapper.CompressionType, readerOptions, @@ -326,7 +350,7 @@ public class TarFactory { if (!IsGenuinelySeekable(stream)) { - return new SharpCompressStream(stream); + return new SharpCompressStream(SharpCompressStream.CreateNonDisposing(stream)); } // Unwrap buffered SharpCompressStream wrappers so we use the real stream's native Seek. @@ -335,35 +359,98 @@ public class TarFactory stream = scs.BaseStream(); } - return new SeekableSharpCompressStream(stream, leaveStreamOpen: false); + return new SeekableSharpCompressStream(SharpCompressStream.CreateNonDisposing(stream)); + } + + private static SharpCompressStream CreateReaderProbeStream(Stream stream, ReaderOptions options) + { + var bufferSize = Math.Max( + options.RewindableBufferSize ?? 0, + TarWrapper.MaximumRewindBufferSize + ); + var probe = CreateNestedRecordingStream(stream); + probe.StartRecording(bufferSize); + return probe; + } + + internal static async ValueTask BufferLZipHeaderAsync( + SharpCompressStream stream, + CancellationToken cancellationToken + ) + { + if (stream is not SeekableSharpCompressStream) + { + return stream; + } + // LZip initializes both its six-byte header and five-byte range coder + // synchronously. Cache those bytes with async I/O before creating it. + var buffered = new SharpCompressStream(stream); + buffered.StartRecording(); + try + { + var header = new byte[11]; + await buffered + .ReadFullyAsync(header, 0, header.Length, cancellationToken) + .ConfigureAwait(false); + buffered.Rewind(); + return buffered; + } + catch + { +#if LEGACY_DOTNET && !NETSTANDARD2_1 + buffered.Dispose(); +#else + await buffered.DisposeAsync().ConfigureAwait(false); +#endif + throw; + } } /// public IReader OpenReader(Stream stream, ReaderOptions? options) { options ??= ReaderOptions.ForExternalStream; - var sharpCompressStream = CreateNestedRecordingStream(stream); - sharpCompressStream.StartRecording(TarWrapper.MaximumRewindBufferSize); - foreach (var wrapper in TarWrapper.Wrappers) + var sharpCompressStream = CreateReaderProbeStream(stream, options); + var transferred = false; + try { - sharpCompressStream.Rewind(); - if (wrapper.IsMatch(sharpCompressStream)) + foreach (var wrapper in TarWrapper.Wrappers) { sharpCompressStream.Rewind(); - var decompressedStream = CreateProbeDecompressionStream( - sharpCompressStream, - wrapper.CompressionType, - options - ); - if (TarArchive.IsTarFile(decompressedStream)) + if (wrapper.IsMatch(sharpCompressStream)) { sharpCompressStream.Rewind(); - sharpCompressStream.StopRecording(); - return new TarReader(sharpCompressStream, options, wrapper.CompressionType); + using var decompressedStream = CreateProbeDecompressionStream( + sharpCompressStream, + wrapper.CompressionType, + options + ); + if (TarArchive.IsTarFile(decompressedStream)) + { + sharpCompressStream.StopRecording(); + var reader = new TarReader( + sharpCompressStream, + options with + { + LeaveStreamOpen = false, + }, + wrapper.CompressionType, + options.LeaveStreamOpen ? null : stream + ); + transferred = true; + return reader; + } } } + throw new InvalidFormatException("Not a tar file."); + } + finally + { + if (!transferred) + { + sharpCompressStream.Dispose(); + } } - throw new InvalidFormatException("Not a tar file."); } /// @@ -375,39 +462,73 @@ public class TarFactory { cancellationToken.ThrowIfCancellationRequested(); options ??= ReaderOptions.ForExternalStream; - var sharpCompressStream = new SharpCompressStream(stream); - sharpCompressStream.StartRecording(TarWrapper.MaximumRewindBufferSize); - foreach (var wrapper in TarWrapper.Wrappers) + var sharpCompressStream = CreateReaderProbeStream(stream, options); + var transferred = false; + try { - sharpCompressStream.Rewind(); - if ( - await wrapper - .IsMatchAsync(sharpCompressStream, cancellationToken) - .ConfigureAwait(false) - ) + foreach (var wrapper in TarWrapper.Wrappers) { sharpCompressStream.Rewind(); - var decompressedStream = await CreateProbeDecompressionStreamAsync( - sharpCompressStream, - wrapper.CompressionType, - options, - cancellationToken - ) - .ConfigureAwait(false); if ( - await TarArchive - .IsTarFileAsync(decompressedStream, cancellationToken) + await wrapper + .IsMatchAsync(sharpCompressStream, cancellationToken) .ConfigureAwait(false) ) { sharpCompressStream.Rewind(); - sharpCompressStream.StopRecording(); - return new TarReader(sharpCompressStream, options, wrapper.CompressionType); + if (wrapper.CompressionType == CompressionType.LZip) + { + sharpCompressStream = await BufferLZipHeaderAsync( + sharpCompressStream, + cancellationToken + ) + .ConfigureAwait(false); + } +#if LEGACY_DOTNET && !NETSTANDARD2_1 + using var decompressedStream = await CreateProbeDecompressionStreamAsync( +#else + await using var decompressedStream = await CreateProbeDecompressionStreamAsync( +#endif + sharpCompressStream, + wrapper.CompressionType, + options, + cancellationToken + ) + .ConfigureAwait(false); + if ( + await TarArchive + .IsTarFileAsync(decompressedStream, cancellationToken) + .ConfigureAwait(false) + ) + { + sharpCompressStream.StopRecording(); + var reader = new TarReader( + sharpCompressStream, + options with + { + LeaveStreamOpen = false, + }, + wrapper.CompressionType, + options.LeaveStreamOpen ? null : stream + ); + transferred = true; + return reader; + } } } + throw new InvalidFormatException("Not a tar file."); + } + finally + { + if (!transferred) + { +#if LEGACY_DOTNET && !NETSTANDARD2_1 + sharpCompressStream.Dispose(); +#else + await sharpCompressStream.DisposeAsync().ConfigureAwait(false); +#endif + } } - - throw new InvalidFormatException("Not a tar file."); } #endregion diff --git a/src/SharpCompress/Readers/ReaderFactory.Async.cs b/src/SharpCompress/Readers/ReaderFactory.Async.cs index 72bd7b09..dd1850bf 100644 --- a/src/SharpCompress/Readers/ReaderFactory.Async.cs +++ b/src/SharpCompress/Readers/ReaderFactory.Async.cs @@ -59,6 +59,13 @@ public static partial class ReaderFactory stream.RequireReadable(); options ??= ReaderOptions.ForExternalStream; + if (options.LeaveStreamOpen) + { + // Internal wrappers must still be disposed when the caller retains its stream. + stream = SharpCompressStream.CreateNonDisposing(stream); + options = options with { LeaveStreamOpen = false }; + } + var sharpCompressStream = SharpCompressStream.Create( stream, bufferSize: options.RewindableBufferSize diff --git a/src/SharpCompress/Readers/ReaderFactory.cs b/src/SharpCompress/Readers/ReaderFactory.cs index 72c4dd71..40e95227 100644 --- a/src/SharpCompress/Readers/ReaderFactory.cs +++ b/src/SharpCompress/Readers/ReaderFactory.cs @@ -32,6 +32,13 @@ public static partial class ReaderFactory stream.RequireReadable(); options ??= ReaderOptions.ForExternalStream; + if (options.LeaveStreamOpen) + { + // Protect the caller's stream, not the internal wrappers and their pooled buffers. + stream = SharpCompressStream.CreateNonDisposing(stream); + options = options with { LeaveStreamOpen = false }; + } + var sharpCompressStream = SharpCompressStream.Create( stream, bufferSize: options.RewindableBufferSize diff --git a/src/SharpCompress/Readers/Tar/TarReader.Factory.cs b/src/SharpCompress/Readers/Tar/TarReader.Factory.cs index aa13c779..931f86b3 100644 --- a/src/SharpCompress/Readers/Tar/TarReader.Factory.cs +++ b/src/SharpCompress/Readers/Tar/TarReader.Factory.cs @@ -30,10 +30,12 @@ public partial class TarReader if (compressionType == CompressionType.GZip) { - return providers.CreateDecompressStream( - compressionType, - nonDisposingStream, - CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options) + return TarFactory.PrepareProbeStream( + providers.CreateDecompressStream( + compressionType, + nonDisposingStream, + CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options) + ) ); } @@ -56,14 +58,18 @@ public partial class TarReader if (compressionType == CompressionType.GZip) { - return await providers - .CreateDecompressStreamAsync( - compressionType, - nonDisposingStream, - CompressionContext.FromStream(nonDisposingStream).WithReaderOptions(options), - cancellationToken - ) - .ConfigureAwait(false); + return TarFactory.PrepareProbeStream( + await providers + .CreateDecompressStreamAsync( + compressionType, + nonDisposingStream, + CompressionContext + .FromStream(nonDisposingStream) + .WithReaderOptions(options), + cancellationToken + ) + .ConfigureAwait(false) + ); } return await providers @@ -90,6 +96,11 @@ public partial class TarReader cancellationToken.ThrowIfCancellationRequested(); stream.NotNull(nameof(stream)); readerOptions ??= ReaderOptions.ForExternalStream; + if (readerOptions.LeaveStreamOpen) + { + stream = SharpCompressStream.CreateNonDisposing(stream); + readerOptions = readerOptions with { LeaveStreamOpen = false }; + } var sharpCompressStream = SharpCompressStream.Create( stream, bufferSize: Math.Max( @@ -97,6 +108,7 @@ public partial class TarReader TarWrapper.MaximumRewindBufferSize ) ); + sharpCompressStream.StartRecording(); long pos = sharpCompressStream.Position; foreach (var wrapper in TarWrapper.Wrappers) { @@ -111,7 +123,18 @@ public partial class TarReader } sharpCompressStream.Position = pos; - var testStream = await CreateProbeDecompressionStreamAsync( + if (wrapper.CompressionType == CompressionType.LZip) + { + sharpCompressStream = await TarFactory + .BufferLZipHeaderAsync(sharpCompressStream, cancellationToken) + .ConfigureAwait(false); + pos = sharpCompressStream.Position; + } +#if LEGACY_DOTNET && !NETSTANDARD2_1 + using var testStream = await CreateProbeDecompressionStreamAsync( +#else + await using var testStream = await CreateProbeDecompressionStreamAsync( +#endif sharpCompressStream, wrapper.CompressionType, readerOptions.Providers, @@ -172,6 +195,11 @@ public partial class TarReader { stream.RequireReadable(); readerOptions ??= ReaderOptions.ForExternalStream; + if (readerOptions.LeaveStreamOpen) + { + stream = SharpCompressStream.CreateNonDisposing(stream); + readerOptions = readerOptions with { LeaveStreamOpen = false }; + } var sharpCompressStream = SharpCompressStream.Create( stream, bufferSize: Math.Max( @@ -179,6 +207,7 @@ public partial class TarReader TarWrapper.MaximumRewindBufferSize ) ); + sharpCompressStream.StartRecording(); long pos = sharpCompressStream.Position; foreach (var wrapper in TarWrapper.Wrappers) { @@ -189,7 +218,7 @@ public partial class TarReader } sharpCompressStream.Position = pos; - var testStream = CreateProbeDecompressionStream( + using var testStream = CreateProbeDecompressionStream( sharpCompressStream, wrapper.CompressionType, readerOptions.Providers, diff --git a/src/SharpCompress/Readers/Tar/TarReader.cs b/src/SharpCompress/Readers/Tar/TarReader.cs index f7c5abba..51a6cf82 100644 --- a/src/SharpCompress/Readers/Tar/TarReader.cs +++ b/src/SharpCompress/Readers/Tar/TarReader.cs @@ -20,16 +20,65 @@ namespace SharpCompress.Readers.Tar; public partial class TarReader : AbstractReader { private readonly CompressionType compressionType; + private readonly Stream? ownedSource; + private bool disposed; - internal TarReader(Stream stream, ReaderOptions options, CompressionType compressionType) + internal TarReader( + Stream stream, + ReaderOptions options, + CompressionType compressionType, + Stream? ownedSource = null + ) : base(options, ArchiveType.Tar) { this.compressionType = compressionType; + this.ownedSource = ownedSource; Volume = new TarVolume(stream, options); } public override TarVolume Volume { get; } + public override void Dispose() + { + if (disposed) + { + return; + } + disposed = true; + try + { + base.Dispose(); + } + finally + { + ownedSource?.Dispose(); + } + } + + public override async ValueTask DisposeAsync() + { + if (disposed) + { + return; + } + disposed = true; + try + { + await base.DisposeAsync().ConfigureAwait(false); + } + finally + { +#if LEGACY_DOTNET && !NETSTANDARD2_1 + ownedSource?.Dispose(); +#else + if (ownedSource is not null) + { + await ownedSource.DisposeAsync().ConfigureAwait(false); + } +#endif + } + } + protected override Stream RequestInitialStream() { var stream = base.RequestInitialStream(); diff --git a/tests/SharpCompress.Test/Streams/TarLifetimeTests.cs b/tests/SharpCompress.Test/Streams/TarLifetimeTests.cs new file mode 100644 index 00000000..4f5a4f0a --- /dev/null +++ b/tests/SharpCompress.Test/Streams/TarLifetimeTests.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.Compressors.Deflate; +using SharpCompress.Factories; +using SharpCompress.IO; +using SharpCompress.Readers; +using SharpCompress.Readers.Tar; +using SharpCompress.Test.Mocks; +using SharpCompress.Writers; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class TarLifetimeTests : TestBase +{ + public static IEnumerable ReaderCases() + { + foreach ( + var filename in new[] + { + "Tar.tar", + "Tar.tar.gz", + "Tar.tar.bz2", + "Tar.tar.zst", + "Tar.tar.lz", + "Tar.tar.xz", + "Tar.tar.Z", + } + ) + { + foreach (var seekable in new[] { false, true }) + { + foreach (var leaveOpen in new[] { false, true }) + { + foreach (var async in new[] { false, true }) + { + foreach (var factory in new[] { 0, 1, 2 }) + { + yield return [filename, seekable, leaveOpen, async, factory]; + } + } + } + } + } + } + + [Theory] + [MemberData(nameof(ReaderCases))] + public async Task Reader_Dispose_Releases_Internal_Buffers( + string filename, + bool seekable, + bool leaveOpen, + bool async, + int factory + ) + { + using var file = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using Stream input = seekable ? file : new ForwardOnlyStream(file); + var options = new ReaderOptions { LeaveStreamOpen = leaveOpen }; + var reader = async + ? (TarReader)await OpenAsyncReader(input, options, factory) + : (TarReader)OpenReader(input, options, factory); + var rings = GetRings(reader.Volume.Stream); + if (seekable && !(async && filename == "Tar.tar.lz")) + { + Assert.Empty(rings); + } + else if (!seekable) + { + Assert.Contains(rings, ring => ring.Capacity >= TarWrapper.MaximumRewindBufferSize); + } + else + { + Assert.NotEmpty(rings); + } + + var entries = 0; + try + { + while ( + async + ? await reader.MoveToNextEntryAsync(CancellationToken.None) + : reader.MoveToNextEntry() + ) + { + if (!reader.Entry.IsDirectory) + { + if (async) + { + await reader.WriteEntryToAsync(Stream.Null, CancellationToken.None); + } + else + { + reader.WriteEntryTo(Stream.Null); + } + entries++; + } + } + } + finally + { + if (async) + { + await reader.DisposeAsync(); + await reader.DisposeAsync(); + } + reader.Dispose(); + } + + Assert.True(entries > 0); + var buffer = typeof(RingBuffer).GetField( + "_buffer", + BindingFlags.Instance | BindingFlags.NonPublic + )!; + Assert.All(rings, ring => Assert.Null(buffer.GetValue(ring))); + Assert.Equal(leaveOpen, input.CanRead); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BZip2_Probe_Preserves_Large_NonSeekable_Block(bool async) + { + var content = new byte[850_000]; + new Random(42).NextBytes(content); + using var archive = new MemoryStream(); + using ( + var writer = WriterFactory.OpenWriter( + archive, + ArchiveType.Tar, + new WriterOptions(CompressionType.BZip2) { LeaveStreamOpen = true } + ) + ) + using (var source = new MemoryStream(content)) + { + writer.Write("random.bin", source); + } + Assert.True(archive.Length > 81920); + archive.Position = 0; + using var input = new ForwardOnlyStream(archive); + if (async) + { + await using var reader = await ReaderFactory.OpenAsyncReader( + input, + cancellationToken: CancellationToken.None + ); + Assert.True(await reader.MoveToNextEntryAsync(CancellationToken.None)); + using var output = new MemoryStream(); + await reader.WriteEntryToAsync(output, CancellationToken.None); + Assert.Equal(content, output.ToArray()); + } + else + { + using var reader = ReaderFactory.OpenReader(input); + Assert.True(reader.MoveToNextEntry()); + using var output = new MemoryStream(); + reader.WriteEntryTo(output); + Assert.Equal(content, output.ToArray()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Failed_Tar_Probe_Does_Not_Close_Source(bool async) + { + using var source = new MemoryStream(); + var content = new byte[1024]; + new Random(42).NextBytes(content); + using ( + var gzip = new GZipStream( + SharpCompressStream.CreateNonDisposing(source), + SharpCompress.Compressors.CompressionMode.Compress + ) + ) + { + gzip.Write(content, 0, content.Length); + } + source.Position = 0; + using var input = new ForwardOnlyStream(source); + var factory = new TarFactory(); + if (async) + { + await Assert.ThrowsAsync(async () => + await factory.OpenAsyncReader( + input, + ReaderOptions.ForExternalStream, + CancellationToken.None + ) + ); + } + else + { + Assert.Throws(() => + factory.OpenReader(input, ReaderOptions.ForExternalStream) + ); + } + Assert.True(input.CanRead); + } + + private static IReader OpenReader(Stream input, ReaderOptions options, int factory) => + factory switch + { + 0 => ReaderFactory.OpenReader(input, options), + 1 => new TarFactory().OpenReader(input, options), + _ => TarReader.OpenReader(input, options), + }; + + private static ValueTask OpenAsyncReader( + Stream input, + ReaderOptions options, + int factory + ) => + factory switch + { + 0 => ReaderFactory.OpenAsyncReader(input, options, CancellationToken.None), + 1 => new TarFactory().OpenAsyncReader(input, options, CancellationToken.None), + _ => TarReader.OpenAsyncReader(input, options, CancellationToken.None), + }; + + private static List GetRings(Stream stream) + { + var rings = new List(); + var field = typeof(SharpCompressStream).GetField( + "_ringBuffer", + BindingFlags.Instance | BindingFlags.NonPublic + )!; + while (stream is SharpCompressStream wrapper) + { + if (field.GetValue(wrapper) is RingBuffer ring) + { + rings.Add(ring); + } + stream = wrapper.BaseStream(); + } + return rings; + } +}