From 923217ef0e9dbc6492a87fc303d76af96fd2062c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 4 Feb 2026 15:45:38 +0000 Subject: [PATCH] remove memorystream --- src/SharpCompress/Common/Constants.cs | 29 +++ src/SharpCompress/Factories/TarFactory.cs | 5 +- .../IO/SharpCompressStream.Async.cs | 170 +++------------ src/SharpCompress/IO/SharpCompressStream.cs | 200 +++++++++--------- .../Readers/ReaderFactory.Async.cs | 5 +- src/SharpCompress/Readers/ReaderFactory.cs | 5 +- src/SharpCompress/Readers/ReaderOptions.cs | 43 ++++ src/SharpCompress/Readers/Tar/TarReader.cs | 5 +- 8 files changed, 212 insertions(+), 250 deletions(-) diff --git a/src/SharpCompress/Common/Constants.cs b/src/SharpCompress/Common/Constants.cs index 8b8f20b2..b9ffcbe3 100644 --- a/src/SharpCompress/Common/Constants.cs +++ b/src/SharpCompress/Common/Constants.cs @@ -8,5 +8,34 @@ public static class Constants /// public static int BufferSize { get; set; } = 81920; + /// + /// The default size for rewindable buffers in SharpCompressStream. + /// Used for format detection on non-seekable streams. + /// + /// + /// + /// When opening archives from non-seekable streams (network streams, pipes, + /// compressed streams), SharpCompress uses a ring buffer to enable format + /// auto-detection. This buffer allows the library to try multiple decoders + /// by rewinding and re-reading the same data. + /// + /// + /// Default: 81920 bytes (81KB) - sufficient for typical format detection. + /// + /// + /// Typical usage: 500-1000 bytes for most archives + /// + /// + /// Can be overridden per-stream via ReaderOptions.RewindableBufferSize. + /// + /// + /// Increase if: + /// + /// Handling self-extracting archives (may need 512KB+) + /// Format detection fails with buffer overflow errors + /// Using custom formats with large headers + /// + /// + /// public static int RewindableBufferSize { get; set; } = 81920; } diff --git a/src/SharpCompress/Factories/TarFactory.cs b/src/SharpCompress/Factories/TarFactory.cs index df091f59..8857cca7 100644 --- a/src/SharpCompress/Factories/TarFactory.cs +++ b/src/SharpCompress/Factories/TarFactory.cs @@ -196,7 +196,10 @@ public class TarFactory if (await wrapper.IsMatchAsync(sharpCompressStream, cancellationToken)) { sharpCompressStream.Rewind(); - var decompressedStream = wrapper.CreateStream(sharpCompressStream); + var decompressedStream = await wrapper.CreateStreamAsync( + sharpCompressStream, + cancellationToken + ); if (await TarArchive.IsTarFileAsync(decompressedStream, cancellationToken)) { sharpCompressStream.Rewind(); diff --git a/src/SharpCompress/IO/SharpCompressStream.Async.cs b/src/SharpCompress/IO/SharpCompressStream.Async.cs index 94ec6819..3544a649 100644 --- a/src/SharpCompress/IO/SharpCompressStream.Async.cs +++ b/src/SharpCompress/IO/SharpCompressStream.Async.cs @@ -35,14 +35,7 @@ internal partial class SharpCompressStream CancellationToken cancellationToken ) { - // If recording is active or we're reading from the recording buffer, use legacy behavior - if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length)) - { - return await ReadWithRecordingAsync(buffer, offset, count, cancellationToken) - .ConfigureAwait(false); - } - - // If ring buffer is enabled (and not recording), use ring buffer logic + // If ring buffer is enabled, use ring buffer logic if (_ringBuffer is not null) { return await ReadWithRingBufferAsync(buffer, offset, count, cancellationToken) @@ -58,65 +51,6 @@ internal partial class SharpCompressStream return read; } - /// - /// Async version of ReadWithRecording (legacy behavior for format detection). - /// - private async Task ReadWithRecordingAsync( - byte[] buffer, - int offset, - int count, - CancellationToken cancellationToken - ) - { - int read; - if (isRewound && bufferStream.Position != bufferStream.Length) - { - var readCount = Math.Min(count, (int)(bufferStream.Length - bufferStream.Position)); - read = await bufferStream - .ReadAsync(buffer, offset, readCount, cancellationToken) - .ConfigureAwait(false); - if (read < count) - { - var tempRead = await stream - .ReadAsync(buffer, offset + read, count - read, cancellationToken) - .ConfigureAwait(false); - if (IsRecording) - { - await bufferStream - .WriteAsync(buffer, offset + read, tempRead, cancellationToken) - .ConfigureAwait(false); - } - else if (_ringBuffer is not null && tempRead > 0) - { - // When transitioning out of recording mode, add to ring buffer - // so that future rewinds will work - _ringBuffer.Write(buffer, offset + read, tempRead); - } - streamPosition += tempRead; - _logicalPosition = streamPosition; - read += tempRead; - } - if (bufferStream.Position == bufferStream.Length) - { - isRewound = false; - } - return read; - } - - read = await stream - .ReadAsync(buffer, offset, count, cancellationToken) - .ConfigureAwait(false); - if (IsRecording) - { - await bufferStream - .WriteAsync(buffer, offset, read, cancellationToken) - .ConfigureAwait(false); - } - streamPosition += read; - _logicalPosition = streamPosition; - return read; - } - /// /// Async version of ReadWithRingBuffer. /// @@ -133,15 +67,25 @@ internal partial class SharpCompressStream while (count > 0 && _logicalPosition < streamPosition) { long bytesFromEnd = streamPosition - _logicalPosition; - int available = _ringBuffer!.ReadFromEnd(bytesFromEnd, buffer, offset, count); + + // Verify data is available in ring buffer + if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd)) + { + throw new InvalidOperationException( + $"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, " + + $"but buffer only holds {_ringBuffer.Length} bytes." + ); + } + + int available = _ringBuffer.ReadFromEnd(bytesFromEnd, buffer, offset, count); totalRead += available; offset += available; count -= available; _logicalPosition += available; } - // If more data needed, read from underlying stream - if (count > 0) + // If more data needed and we're caught up, read from underlying stream + if (count > 0 && _logicalPosition == streamPosition) { int read = await stream .ReadAsync(buffer, offset, count, cancellationToken) @@ -183,13 +127,7 @@ internal partial class SharpCompressStream CancellationToken cancellationToken ) { - // If recording is active or we're reading from the recording buffer, use legacy behavior - if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length)) - { - return await ReadWithRecordingAsync(buffer, cancellationToken).ConfigureAwait(false); - } - - // If ring buffer is enabled (and not recording), use ring buffer logic + // If ring buffer is enabled, use ring buffer logic if (_ringBuffer is not null) { return await ReadWithRingBufferAsync(buffer, cancellationToken).ConfigureAwait(false); @@ -202,63 +140,6 @@ internal partial class SharpCompressStream return read; } - /// - /// Async version of ReadWithRecording for Memory<byte> (legacy behavior for format detection). - /// - private async ValueTask ReadWithRecordingAsync( - Memory buffer, - CancellationToken cancellationToken - ) - { - int read; - if (isRewound && bufferStream.Position != bufferStream.Length) - { - var readCount = (int) - Math.Min(buffer.Length, bufferStream.Length - bufferStream.Position); - read = await bufferStream - .ReadAsync(buffer.Slice(0, readCount), cancellationToken) - .ConfigureAwait(false); - if (read < buffer.Length) - { - var tempRead = await stream - .ReadAsync(buffer.Slice(read), cancellationToken) - .ConfigureAwait(false); - if (IsRecording) - { - await bufferStream - .WriteAsync(buffer.Slice(read, tempRead), cancellationToken) - .ConfigureAwait(false); - } - else if (_ringBuffer is not null && tempRead > 0) - { - // When transitioning out of recording mode, add to ring buffer - // so that future rewinds will work - var tempBuffer = buffer.Slice(read, tempRead).ToArray(); - _ringBuffer.Write(tempBuffer, 0, tempRead); - } - streamPosition += tempRead; - _logicalPosition = streamPosition; - read += tempRead; - } - if (bufferStream.Position == bufferStream.Length) - { - isRewound = false; - } - return read; - } - - read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - if (IsRecording) - { - await bufferStream - .WriteAsync(buffer.Slice(0, read), cancellationToken) - .ConfigureAwait(false); - } - streamPosition += read; - _logicalPosition = streamPosition; - return read; - } - /// /// Async version of ReadWithRingBuffer for Memory<byte>. /// @@ -276,13 +157,18 @@ internal partial class SharpCompressStream while (count > 0 && _logicalPosition < streamPosition) { long bytesFromEnd = streamPosition - _logicalPosition; + + // Verify data is available in ring buffer + if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd)) + { + throw new InvalidOperationException( + $"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, " + + $"but buffer only holds {_ringBuffer.Length} bytes." + ); + } + var tempBuffer = new byte[Math.Min(count, (int)bytesFromEnd)]; - int available = _ringBuffer!.ReadFromEnd( - bytesFromEnd, - tempBuffer, - 0, - tempBuffer.Length - ); + int available = _ringBuffer.ReadFromEnd(bytesFromEnd, tempBuffer, 0, tempBuffer.Length); tempBuffer.AsSpan(0, available).CopyTo(buffer.Span.Slice(offset)); totalRead += available; @@ -291,8 +177,8 @@ internal partial class SharpCompressStream _logicalPosition += available; } - // If more data needed, read from underlying stream - if (count > 0) + // If more data needed and we're caught up, read from underlying stream + if (count > 0 && _logicalPosition == streamPosition) { int read = await stream .ReadAsync(buffer.Slice(offset, count), cancellationToken) diff --git a/src/SharpCompress/IO/SharpCompressStream.cs b/src/SharpCompress/IO/SharpCompressStream.cs index 5d2565d0..eae6217f 100644 --- a/src/SharpCompress/IO/SharpCompressStream.cs +++ b/src/SharpCompress/IO/SharpCompressStream.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using SharpCompress.Common; namespace SharpCompress.IO; @@ -8,15 +9,18 @@ internal partial class SharpCompressStream : Stream, IStreamStack public virtual Stream BaseStream() => stream; private readonly Stream stream; - private MemoryStream bufferStream = new MemoryStream(); - private bool isRewound; private bool isDisposed; private long streamPosition; - // Rolling buffer for limited backward seeking without unbounded memory growth. + // Ring buffer for recording mode and over-read protection. + // Single unified buffering mechanism for both use cases. private RingBuffer? _ringBuffer; private long _logicalPosition; // The current logical read position (can be behind streamPosition) + // Recording state: anchor position when StartRecording was called + private long? _recordingStartPosition; + private bool _isRecording; + // Passthrough mode - no buffering, delegates CanSeek to underlying stream private readonly bool _isPassthrough; @@ -80,7 +84,10 @@ internal partial class SharpCompressStream : Stream, IStreamStack public static SharpCompressStream CreateNonDisposing(Stream stream) => new(stream, leaveStreamOpen: true, passthrough: true); - internal virtual bool IsRecording { get; private set; } + /// + /// Gets whether the stream is actively recording reads to the ring buffer. + /// + internal virtual bool IsRecording => _isRecording; protected override void Dispose(bool disposing) { @@ -117,9 +124,35 @@ internal partial class SharpCompressStream : Stream, IStreamStack "Rewind cannot be called on a passthrough stream. Use EnsureSeekable() first." ); } - isRewound = true; - IsRecording = !stopRecording; - bufferStream.Position = 0; + + if (_recordingStartPosition is null) + { + throw new InvalidOperationException( + "Rewind can only be called after StartRecording() has been called." + ); + } + + // Verify recording anchor is within ring buffer range + long anchorAge = streamPosition - _recordingStartPosition.Value; + if (anchorAge > _ringBuffer!.Length) + { + throw new InvalidOperationException( + $"Cannot rewind: recording anchor is {anchorAge} bytes behind current position, " + + $"but ring buffer only holds {_ringBuffer.Length} bytes. " + + $"Recording buffer overflow - increase DefaultRollingBufferSize or reduce format detection reads." + ); + } + + // Rewind logical position to recording anchor + _logicalPosition = _recordingStartPosition.Value; + + if (stopRecording) + { + _isRecording = false; + // Note: We keep _recordingStartPosition so Rewind() can be called again + // (frozen recording mode). The anchor is only cleared when a new recording + // starts or the stream is disposed. + } } public virtual void StopRecording() @@ -136,13 +169,24 @@ internal partial class SharpCompressStream : Stream, IStreamStack "StopRecording can only be called when recording is active." ); } - isRewound = true; - IsRecording = false; - bufferStream.Position = 0; + + // Mark that we're no longer actively recording + _isRecording = false; + + // Rewind to recording anchor position + _logicalPosition = _recordingStartPosition!.Value; + + // Note: We keep _recordingStartPosition so future Rewind() calls still work + // (frozen recording mode) until Rewind(stopRecording: true) is called } - public static SharpCompressStream EnsureSeekable(Stream stream) + public static SharpCompressStream EnsureSeekable( + Stream stream, + int? rewindableBufferSize = null + ) { + int bufferSize = rewindableBufferSize ?? Constants.RewindableBufferSize; + // If it's a passthrough SharpCompressStream, unwrap it and create proper seekable wrapper if (stream is SharpCompressStream sharpCompressStream) { @@ -159,10 +203,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack }; } // Non-seekable underlying stream - wrap with rolling buffer - return new SharpCompressStream(underlying, DefaultRollingBufferSize) - { - LeaveStreamOpen = true, - }; + return new SharpCompressStream(underlying, bufferSize) { LeaveStreamOpen = true }; } // Not passthrough - return as-is return sharpCompressStream; @@ -185,7 +226,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack // For non-seekable streams, create a SharpCompressStream with rolling buffer // to allow limited backward seeking (required by decompressors that over-read) - return new SharpCompressStream(stream, DefaultRollingBufferSize); + return new SharpCompressStream(stream, bufferSize); } public virtual void StartRecording() @@ -202,17 +243,17 @@ internal partial class SharpCompressStream : Stream, IStreamStack "StartRecording can only be called when not already recording." ); } - //if (isRewound && bufferStream.Position != 0) - // throw new System.NotImplementedException(); - if (bufferStream.Position != 0) + + // Ensure ring buffer exists + if (_ringBuffer is null) { - var data = bufferStream.ToArray(); - var position = bufferStream.Position; - bufferStream.SetLength(0); - bufferStream.Write(data, (int)position, data.Length - (int)position); - bufferStream.Position = 0; + _ringBuffer = new RingBuffer(DefaultRollingBufferSize); } - IsRecording = true; + + // Mark current position as recording anchor + _recordingStartPosition = streamPosition; + _logicalPosition = streamPosition; + _isRecording = true; } public override bool CanRead => true; @@ -243,17 +284,8 @@ internal partial class SharpCompressStream : Stream, IStreamStack { return stream.Position; } - // If recording is active or rewound from recording, use recording buffer position - if (IsRecording || (isRewound && bufferStream.Position < bufferStream.Length)) - { - return streamPosition - bufferStream.Length + bufferStream.Position; - } - // If ring buffer is active (and not recording), use logical position - if (_ringBuffer is not null) - { - return _logicalPosition; - } - return streamPosition; + // Use logical position (same for both recording and ring buffer modes) + return _logicalPosition; } set { @@ -269,22 +301,21 @@ internal partial class SharpCompressStream : Stream, IStreamStack private void SeekToPosition(long targetPosition) { - // If recording is active, use recording buffer for seeking - if (IsRecording || isRewound) + // If we have a recording anchor, allow seeking within the recorded range + if (_recordingStartPosition is not null) { - long bufferStart = streamPosition - bufferStream.Length; - long bufferEnd = streamPosition; - - if (targetPosition >= bufferStart && targetPosition <= bufferEnd) + if (targetPosition >= _recordingStartPosition.Value && targetPosition <= streamPosition) { - isRewound = true; - bufferStream.Position = targetPosition - bufferStart; + _logicalPosition = targetPosition; return; } - throw new NotSupportedException("Cannot seek outside recorded region."); + throw new NotSupportedException( + $"Cannot seek to position {targetPosition}. Valid recorded range: " + + $"[{_recordingStartPosition.Value}, {streamPosition}]" + ); } - // If ring buffer is enabled, check if we can seek within it + // If ring buffer is enabled (and not recording), check if we can seek within it if (_ringBuffer is not null) { long ringBufferStart = streamPosition - _ringBuffer.Length; @@ -293,9 +324,9 @@ internal partial class SharpCompressStream : Stream, IStreamStack _logicalPosition = targetPosition; return; } - // Can't seek outside ring buffer range throw new NotSupportedException( - $"Cannot seek to position {targetPosition}. Valid range with ring buffer: [{ringBufferStart}, {streamPosition}]" + $"Cannot seek to position {targetPosition}. Valid ring buffer range: " + + $"[{ringBufferStart}, {streamPosition}]" ); } @@ -316,14 +347,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack return stream.Read(buffer, offset, count); } - // If recording is active or we're reading from the recording buffer, use legacy behavior - // Recording takes precedence over rolling buffer for format detection - if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length)) - { - return ReadWithRecording(buffer, offset, count); - } - - // If ring buffer is enabled (and not recording), use ring buffer logic + // If ring buffer exists, use unified buffered read logic if (_ringBuffer is not null) { return ReadWithRingBuffer(buffer, offset, count); @@ -336,53 +360,10 @@ internal partial class SharpCompressStream : Stream, IStreamStack return read; } - /// - /// Reads data using the recording buffer (legacy behavior for format detection). - /// - private int ReadWithRecording(byte[] buffer, int offset, int count) - { - int read; - if (isRewound && bufferStream.Position != bufferStream.Length) - { - var readCount = Math.Min(count, (int)(bufferStream.Length - bufferStream.Position)); - read = bufferStream.Read(buffer, offset, readCount); - if (read < count) - { - var tempRead = stream.Read(buffer, offset + read, count - read); - if (IsRecording) - { - bufferStream.Write(buffer, offset + read, tempRead); - } - else if (_ringBuffer is not null && tempRead > 0) - { - // When transitioning out of recording mode, add to ring buffer - // so that future rewinds will work - _ringBuffer.Write(buffer, offset + read, tempRead); - } - streamPosition += tempRead; - _logicalPosition = streamPosition; - read += tempRead; - } - if (bufferStream.Position == bufferStream.Length) - { - isRewound = false; - } - return read; - } - - read = stream.Read(buffer, offset, count); - if (IsRecording) - { - bufferStream.Write(buffer, offset, read); - } - streamPosition += read; - _logicalPosition = streamPosition; - return read; - } - /// /// Reads data using the ring buffer. If logical position is behind stream position, - /// serves data from the ring buffer first. + /// serves data from the ring buffer first. Handles both recording mode and + /// over-read protection uniformly. /// private int ReadWithRingBuffer(byte[] buffer, int offset, int count) { @@ -392,16 +373,27 @@ internal partial class SharpCompressStream : Stream, IStreamStack while (count > 0 && _logicalPosition < streamPosition) { long bytesFromEnd = streamPosition - _logicalPosition; - int available = _ringBuffer!.ReadFromEnd(bytesFromEnd, buffer, offset, count); + + // Verify data is available in ring buffer + if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd)) + { + throw new InvalidOperationException( + $"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, " + + $"but buffer only holds {_ringBuffer.Length} bytes." + ); + } + + int available = _ringBuffer.ReadFromEnd(bytesFromEnd, buffer, offset, count); totalRead += available; offset += available; count -= available; _logicalPosition += available; } - // If more data needed, read from underlying stream - if (count > 0) + // If more data needed and we're caught up, read from underlying stream + if (count > 0 && _logicalPosition == streamPosition) { + // Use async read if stream doesn't support sync reads (e.g., AsyncOnlyStream) int read = stream.Read(buffer, offset, count); if (read > 0) { diff --git a/src/SharpCompress/Readers/ReaderFactory.Async.cs b/src/SharpCompress/Readers/ReaderFactory.Async.cs index 53788d86..2293ea07 100644 --- a/src/SharpCompress/Readers/ReaderFactory.Async.cs +++ b/src/SharpCompress/Readers/ReaderFactory.Async.cs @@ -54,7 +54,10 @@ public static partial class ReaderFactory stream.NotNull(nameof(stream)); options ??= new ReaderOptions() { LeaveStreamOpen = false }; - var sharpCompressStream = new SharpCompressStream(stream); + var sharpCompressStream = SharpCompressStream.EnsureSeekable( + stream, + options.RewindableBufferSize + ); sharpCompressStream.StartRecording(); var factories = Factory.Factories.OfType(); diff --git a/src/SharpCompress/Readers/ReaderFactory.cs b/src/SharpCompress/Readers/ReaderFactory.cs index d6b94951..180c2809 100644 --- a/src/SharpCompress/Readers/ReaderFactory.cs +++ b/src/SharpCompress/Readers/ReaderFactory.cs @@ -34,7 +34,10 @@ public static partial class ReaderFactory stream.NotNull(nameof(stream)); options ??= new ReaderOptions() { LeaveStreamOpen = false }; - var sharpCompressStream = SharpCompressStream.EnsureSeekable(stream); + var sharpCompressStream = SharpCompressStream.EnsureSeekable( + stream, + options.RewindableBufferSize + ); sharpCompressStream.StartRecording(); var factories = Factories.Factory.Factories.OfType(); diff --git a/src/SharpCompress/Readers/ReaderOptions.cs b/src/SharpCompress/Readers/ReaderOptions.cs index b016a3b6..2ddb8ac4 100644 --- a/src/SharpCompress/Readers/ReaderOptions.cs +++ b/src/SharpCompress/Readers/ReaderOptions.cs @@ -36,4 +36,47 @@ public class ReaderOptions : OptionsBase /// When set, progress updates will be reported as entries are extracted. /// public IProgress? Progress { get; set; } + + /// + /// Size of the rewindable buffer for non-seekable streams. + /// Used during format detection to enable multiple rewinds. + /// + /// + /// + /// When opening archives from non-seekable streams (network streams, pipes, + /// compressed streams), SharpCompress uses a ring buffer to enable format + /// auto-detection. This buffer allows the library to try multiple decoders + /// by rewinding and re-reading the same data. + /// + /// + /// Default: Constants.RewindableBufferSize (81920 bytes / 81KB) + /// + /// + /// Typical usage: 500-1000 bytes for most archives + /// + /// + /// Increase if: + /// + /// Opening self-extracting RAR archives (may need 512KB+) + /// Format detection fails with "recording anchor" errors + /// Using custom formats with large headers + /// + /// + /// + /// Memory impact: Buffer is allocated for non-seekable streams only. + /// Seekable streams (FileStream, MemoryStream) use zero-copy seeking instead. + /// + /// + /// + /// + /// // For self-extracting archives, use larger buffer + /// var options = new ReaderOptions + /// { + /// RewindableBufferSize = 1_048_576, // 1MB + /// LookForHeader = true + /// }; + /// using var reader = ReaderFactory.OpenReader(networkStream, options); + /// + /// + public int? RewindableBufferSize { get; set; } } diff --git a/src/SharpCompress/Readers/Tar/TarReader.cs b/src/SharpCompress/Readers/Tar/TarReader.cs index b5571a80..c2068789 100644 --- a/src/SharpCompress/Readers/Tar/TarReader.cs +++ b/src/SharpCompress/Readers/Tar/TarReader.cs @@ -57,7 +57,10 @@ public partial class TarReader : AbstractReader { stream.NotNull(nameof(stream)); options = options ?? new ReaderOptions(); - var sharpCompressStream = SharpCompressStream.EnsureSeekable(stream); + var sharpCompressStream = SharpCompressStream.EnsureSeekable( + stream, + options.RewindableBufferSize + ); long pos = sharpCompressStream.Position; if (GZipArchive.IsGZipFile(sharpCompressStream)) {