starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-11-15 17:02:03 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

remove memorystream

Browse Source
This commit is contained in:
Adam Hathcock
2026-02-04 15:45:38 +00:00
parent 0d81d7f243
commit 923217ef0e
8 changed files with 212 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/SharpCompress/Common/Constants.cs
View File

@@ -8,5 +8,34 @@ public static class Constants
/// </summary>
public static int BufferSize { get; set; } = 81920;
/// <summary>
/// The default size for rewindable buffers in SharpCompressStream.
/// Used for format detection on non-seekable streams.
/// </summary>
/// <remarks>
/// <para>
/// When opening archives from non-seekable streams (network streams, pipes,
/// compressed streams), SharpCompress uses a ring buffer to enable format
/// auto-detection. This buffer allows the library to try multiple decoders
/// by rewinding and re-reading the same data.
/// </para>
/// <para>
/// <b>Default:</b> 81920 bytes (81KB) - sufficient for typical format detection.
/// </para>
/// <para>
/// <b>Typical usage:</b> 500-1000 bytes for most archives
/// </para>
/// <para>
/// <b>Can be overridden per-stream via ReaderOptions.RewindableBufferSize.</b>
/// </para>
/// <para>
/// <b>Increase if:</b>
/// <list type="bullet">
/// <item>Handling self-extracting archives (may need 512KB+)</item>
/// <item>Format detection fails with buffer overflow errors</item>
/// <item>Using custom formats with large headers</item>
/// </list>
/// </para>
/// </remarks>
public static int RewindableBufferSize { get; set; } = 81920;
}

5
src/SharpCompress/Factories/TarFactory.cs
View File

@@ -196,7 +196,10 @@ public class TarFactory
if (await wrapper.IsMatchAsync(sharpCompressStream, cancellationToken))
{
sharpCompressStream.Rewind();
var decompressedStream = wrapper.CreateStream(sharpCompressStream);
var decompressedStream = await wrapper.CreateStreamAsync(
sharpCompressStream,
cancellationToken
);
if (await TarArchive.IsTarFileAsync(decompressedStream, cancellationToken))
{
sharpCompressStream.Rewind();

170
src/SharpCompress/IO/SharpCompressStream.Async.cs
View File

@@ -35,14 +35,7 @@ internal partial class SharpCompressStream
CancellationToken cancellationToken
)
{
// If recording is active or we're reading from the recording buffer, use legacy behavior
if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length))
{
return await ReadWithRecordingAsync(buffer, offset, count, cancellationToken)
.ConfigureAwait(false);
}
// If ring buffer is enabled (and not recording), use ring buffer logic
// If ring buffer is enabled, use ring buffer logic
if (_ringBuffer is not null)
{
return await ReadWithRingBufferAsync(buffer, offset, count, cancellationToken)
@@ -58,65 +51,6 @@ internal partial class SharpCompressStream
return read;
}
/// <summary>
/// Async version of ReadWithRecording (legacy behavior for format detection).
/// </summary>
private async Task<int> ReadWithRecordingAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
int read;
if (isRewound && bufferStream.Position != bufferStream.Length)
{
var readCount = Math.Min(count, (int)(bufferStream.Length - bufferStream.Position));
read = await bufferStream
.ReadAsync(buffer, offset, readCount, cancellationToken)
.ConfigureAwait(false);
if (read < count)
{
var tempRead = await stream
.ReadAsync(buffer, offset + read, count - read, cancellationToken)
.ConfigureAwait(false);
if (IsRecording)
{
await bufferStream
.WriteAsync(buffer, offset + read, tempRead, cancellationToken)
.ConfigureAwait(false);
}
else if (_ringBuffer is not null && tempRead > 0)
{
// When transitioning out of recording mode, add to ring buffer
// so that future rewinds will work
_ringBuffer.Write(buffer, offset + read, tempRead);
}
streamPosition += tempRead;
_logicalPosition = streamPosition;
read += tempRead;
}
if (bufferStream.Position == bufferStream.Length)
{
isRewound = false;
}
return read;
}
read = await stream
.ReadAsync(buffer, offset, count, cancellationToken)
.ConfigureAwait(false);
if (IsRecording)
{
await bufferStream
.WriteAsync(buffer, offset, read, cancellationToken)
.ConfigureAwait(false);
}
streamPosition += read;
_logicalPosition = streamPosition;
return read;
}
/// <summary>
/// Async version of ReadWithRingBuffer.
/// </summary>
@@ -133,15 +67,25 @@ internal partial class SharpCompressStream
while (count > 0 && _logicalPosition < streamPosition)
{
long bytesFromEnd = streamPosition - _logicalPosition;
int available = _ringBuffer!.ReadFromEnd(bytesFromEnd, buffer, offset, count);
// Verify data is available in ring buffer
if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd))
{
throw new InvalidOperationException(
$"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, "
+ $"but buffer only holds {_ringBuffer.Length} bytes."
);
}
int available = _ringBuffer.ReadFromEnd(bytesFromEnd, buffer, offset, count);
totalRead += available;
offset += available;
count -= available;
_logicalPosition += available;
}
// If more data needed, read from underlying stream
if (count > 0)
// If more data needed and we're caught up, read from underlying stream
if (count > 0 && _logicalPosition == streamPosition)
{
int read = await stream
.ReadAsync(buffer, offset, count, cancellationToken)
@@ -183,13 +127,7 @@ internal partial class SharpCompressStream
CancellationToken cancellationToken
)
{
// If recording is active or we're reading from the recording buffer, use legacy behavior
if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length))
{
return await ReadWithRecordingAsync(buffer, cancellationToken).ConfigureAwait(false);
}
// If ring buffer is enabled (and not recording), use ring buffer logic
// If ring buffer is enabled, use ring buffer logic
if (_ringBuffer is not null)
{
return await ReadWithRingBufferAsync(buffer, cancellationToken).ConfigureAwait(false);
@@ -202,63 +140,6 @@ internal partial class SharpCompressStream
return read;
}
/// <summary>
/// Async version of ReadWithRecording for Memory&lt;byte&gt; (legacy behavior for format detection).
/// </summary>
private async ValueTask<int> ReadWithRecordingAsync(
Memory<byte> buffer,
CancellationToken cancellationToken
)
{
int read;
if (isRewound && bufferStream.Position != bufferStream.Length)
{
var readCount = (int)
Math.Min(buffer.Length, bufferStream.Length - bufferStream.Position);
read = await bufferStream
.ReadAsync(buffer.Slice(0, readCount), cancellationToken)
.ConfigureAwait(false);
if (read < buffer.Length)
{
var tempRead = await stream
.ReadAsync(buffer.Slice(read), cancellationToken)
.ConfigureAwait(false);
if (IsRecording)
{
await bufferStream
.WriteAsync(buffer.Slice(read, tempRead), cancellationToken)
.ConfigureAwait(false);
}
else if (_ringBuffer is not null && tempRead > 0)
{
// When transitioning out of recording mode, add to ring buffer
// so that future rewinds will work
var tempBuffer = buffer.Slice(read, tempRead).ToArray();
_ringBuffer.Write(tempBuffer, 0, tempRead);
}
streamPosition += tempRead;
_logicalPosition = streamPosition;
read += tempRead;
}
if (bufferStream.Position == bufferStream.Length)
{
isRewound = false;
}
return read;
}
read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
if (IsRecording)
{
await bufferStream
.WriteAsync(buffer.Slice(0, read), cancellationToken)
.ConfigureAwait(false);
}
streamPosition += read;
_logicalPosition = streamPosition;
return read;
}
/// <summary>
/// Async version of ReadWithRingBuffer for Memory&lt;byte&gt;.
/// </summary>
@@ -276,13 +157,18 @@ internal partial class SharpCompressStream
while (count > 0 && _logicalPosition < streamPosition)
{
long bytesFromEnd = streamPosition - _logicalPosition;
// Verify data is available in ring buffer
if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd))
{
throw new InvalidOperationException(
$"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, "
+ $"but buffer only holds {_ringBuffer.Length} bytes."
);
}
var tempBuffer = new byte[Math.Min(count, (int)bytesFromEnd)];
int available = _ringBuffer!.ReadFromEnd(
bytesFromEnd,
tempBuffer,
0,
tempBuffer.Length
);
int available = _ringBuffer.ReadFromEnd(bytesFromEnd, tempBuffer, 0, tempBuffer.Length);
tempBuffer.AsSpan(0, available).CopyTo(buffer.Span.Slice(offset));
totalRead += available;
@@ -291,8 +177,8 @@ internal partial class SharpCompressStream
_logicalPosition += available;
}
// If more data needed, read from underlying stream
if (count > 0)
// If more data needed and we're caught up, read from underlying stream
if (count > 0 && _logicalPosition == streamPosition)
{
int read = await stream
.ReadAsync(buffer.Slice(offset, count), cancellationToken)

200
src/SharpCompress/IO/SharpCompressStream.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.IO;
using SharpCompress.Common;
namespace SharpCompress.IO;
@@ -8,15 +9,18 @@ internal partial class SharpCompressStream : Stream, IStreamStack
public virtual Stream BaseStream() => stream;
private readonly Stream stream;
private MemoryStream bufferStream = new MemoryStream();
private bool isRewound;
private bool isDisposed;
private long streamPosition;
// Rolling buffer for limited backward seeking without unbounded memory growth.
// Ring buffer for recording mode and over-read protection.
// Single unified buffering mechanism for both use cases.
private RingBuffer? _ringBuffer;
private long _logicalPosition; // The current logical read position (can be behind streamPosition)
// Recording state: anchor position when StartRecording was called
private long? _recordingStartPosition;
private bool _isRecording;
// Passthrough mode - no buffering, delegates CanSeek to underlying stream
private readonly bool _isPassthrough;
@@ -80,7 +84,10 @@ internal partial class SharpCompressStream : Stream, IStreamStack
public static SharpCompressStream CreateNonDisposing(Stream stream) =>
new(stream, leaveStreamOpen: true, passthrough: true);
internal virtual bool IsRecording { get; private set; }
/// <summary>
/// Gets whether the stream is actively recording reads to the ring buffer.
/// </summary>
internal virtual bool IsRecording => _isRecording;
protected override void Dispose(bool disposing)
{
@@ -117,9 +124,35 @@ internal partial class SharpCompressStream : Stream, IStreamStack
"Rewind cannot be called on a passthrough stream. Use EnsureSeekable() first."
);
}
isRewound = true;
IsRecording = !stopRecording;
bufferStream.Position = 0;
if (_recordingStartPosition is null)
{
throw new InvalidOperationException(
"Rewind can only be called after StartRecording() has been called."
);
}
// Verify recording anchor is within ring buffer range
long anchorAge = streamPosition - _recordingStartPosition.Value;
if (anchorAge > _ringBuffer!.Length)
{
throw new InvalidOperationException(
$"Cannot rewind: recording anchor is {anchorAge} bytes behind current position, "
+ $"but ring buffer only holds {_ringBuffer.Length} bytes. "
+ $"Recording buffer overflow - increase DefaultRollingBufferSize or reduce format detection reads."
);
}
// Rewind logical position to recording anchor
_logicalPosition = _recordingStartPosition.Value;
if (stopRecording)
{
_isRecording = false;
// Note: We keep _recordingStartPosition so Rewind() can be called again
// (frozen recording mode). The anchor is only cleared when a new recording
// starts or the stream is disposed.
}
}
public virtual void StopRecording()
@@ -136,13 +169,24 @@ internal partial class SharpCompressStream : Stream, IStreamStack
"StopRecording can only be called when recording is active."
);
}
isRewound = true;
IsRecording = false;
bufferStream.Position = 0;
// Mark that we're no longer actively recording
_isRecording = false;
// Rewind to recording anchor position
_logicalPosition = _recordingStartPosition!.Value;
// Note: We keep _recordingStartPosition so future Rewind() calls still work
// (frozen recording mode) until Rewind(stopRecording: true) is called
}
public static SharpCompressStream EnsureSeekable(Stream stream)
public static SharpCompressStream EnsureSeekable(
Stream stream,
int? rewindableBufferSize = null
)
{
int bufferSize = rewindableBufferSize ?? Constants.RewindableBufferSize;
// If it's a passthrough SharpCompressStream, unwrap it and create proper seekable wrapper
if (stream is SharpCompressStream sharpCompressStream)
{
@@ -159,10 +203,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack
};
}
// Non-seekable underlying stream - wrap with rolling buffer
return new SharpCompressStream(underlying, DefaultRollingBufferSize)
{
LeaveStreamOpen = true,
};
return new SharpCompressStream(underlying, bufferSize) { LeaveStreamOpen = true };
}
// Not passthrough - return as-is
return sharpCompressStream;
@@ -185,7 +226,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack
// For non-seekable streams, create a SharpCompressStream with rolling buffer
// to allow limited backward seeking (required by decompressors that over-read)
return new SharpCompressStream(stream, DefaultRollingBufferSize);
return new SharpCompressStream(stream, bufferSize);
}
public virtual void StartRecording()
@@ -202,17 +243,17 @@ internal partial class SharpCompressStream : Stream, IStreamStack
"StartRecording can only be called when not already recording."
);
}
//if (isRewound && bufferStream.Position != 0)
// throw new System.NotImplementedException();
if (bufferStream.Position != 0)
// Ensure ring buffer exists
if (_ringBuffer is null)
{
var data = bufferStream.ToArray();
var position = bufferStream.Position;
bufferStream.SetLength(0);
bufferStream.Write(data, (int)position, data.Length - (int)position);
bufferStream.Position = 0;
_ringBuffer = new RingBuffer(DefaultRollingBufferSize);
}
IsRecording = true;
// Mark current position as recording anchor
_recordingStartPosition = streamPosition;
_logicalPosition = streamPosition;
_isRecording = true;
}
public override bool CanRead => true;
@@ -243,17 +284,8 @@ internal partial class SharpCompressStream : Stream, IStreamStack
{
return stream.Position;
}
// If recording is active or rewound from recording, use recording buffer position
if (IsRecording || (isRewound && bufferStream.Position < bufferStream.Length))
{
return streamPosition - bufferStream.Length + bufferStream.Position;
}
// If ring buffer is active (and not recording), use logical position
if (_ringBuffer is not null)
{
return _logicalPosition;
}
return streamPosition;
// Use logical position (same for both recording and ring buffer modes)
return _logicalPosition;
}
set
{
@@ -269,22 +301,21 @@ internal partial class SharpCompressStream : Stream, IStreamStack
private void SeekToPosition(long targetPosition)
{
// If recording is active, use recording buffer for seeking
if (IsRecording || isRewound)
// If we have a recording anchor, allow seeking within the recorded range
if (_recordingStartPosition is not null)
{
long bufferStart = streamPosition - bufferStream.Length;
long bufferEnd = streamPosition;
if (targetPosition >= bufferStart && targetPosition <= bufferEnd)
if (targetPosition >= _recordingStartPosition.Value && targetPosition <= streamPosition)
{
isRewound = true;
bufferStream.Position = targetPosition - bufferStart;
_logicalPosition = targetPosition;
return;
}
throw new NotSupportedException("Cannot seek outside recorded region.");
throw new NotSupportedException(
$"Cannot seek to position {targetPosition}. Valid recorded range: "
+ $"[{_recordingStartPosition.Value}, {streamPosition}]"
);
}
// If ring buffer is enabled, check if we can seek within it
// If ring buffer is enabled (and not recording), check if we can seek within it
if (_ringBuffer is not null)
{
long ringBufferStart = streamPosition - _ringBuffer.Length;
@@ -293,9 +324,9 @@ internal partial class SharpCompressStream : Stream, IStreamStack
_logicalPosition = targetPosition;
return;
}
// Can't seek outside ring buffer range
throw new NotSupportedException(
$"Cannot seek to position {targetPosition}. Valid range with ring buffer: [{ringBufferStart}, {streamPosition}]"
$"Cannot seek to position {targetPosition}. Valid ring buffer range: "
+ $"[{ringBufferStart}, {streamPosition}]"
);
}
@@ -316,14 +347,7 @@ internal partial class SharpCompressStream : Stream, IStreamStack
return stream.Read(buffer, offset, count);
}
// If recording is active or we're reading from the recording buffer, use legacy behavior
// Recording takes precedence over rolling buffer for format detection
if (IsRecording || (isRewound && bufferStream.Position != bufferStream.Length))
{
return ReadWithRecording(buffer, offset, count);
}
// If ring buffer is enabled (and not recording), use ring buffer logic
// If ring buffer exists, use unified buffered read logic
if (_ringBuffer is not null)
{
return ReadWithRingBuffer(buffer, offset, count);
@@ -336,53 +360,10 @@ internal partial class SharpCompressStream : Stream, IStreamStack
return read;
}
/// <summary>
/// Reads data using the recording buffer (legacy behavior for format detection).
/// </summary>
private int ReadWithRecording(byte[] buffer, int offset, int count)
{
int read;
if (isRewound && bufferStream.Position != bufferStream.Length)
{
var readCount = Math.Min(count, (int)(bufferStream.Length - bufferStream.Position));
read = bufferStream.Read(buffer, offset, readCount);
if (read < count)
{
var tempRead = stream.Read(buffer, offset + read, count - read);
if (IsRecording)
{
bufferStream.Write(buffer, offset + read, tempRead);
}
else if (_ringBuffer is not null && tempRead > 0)
{
// When transitioning out of recording mode, add to ring buffer
// so that future rewinds will work
_ringBuffer.Write(buffer, offset + read, tempRead);
}
streamPosition += tempRead;
_logicalPosition = streamPosition;
read += tempRead;
}
if (bufferStream.Position == bufferStream.Length)
{
isRewound = false;
}
return read;
}
read = stream.Read(buffer, offset, count);
if (IsRecording)
{
bufferStream.Write(buffer, offset, read);
}
streamPosition += read;
_logicalPosition = streamPosition;
return read;
}
/// <summary>
/// Reads data using the ring buffer. If logical position is behind stream position,
/// serves data from the ring buffer first.
/// serves data from the ring buffer first. Handles both recording mode and
/// over-read protection uniformly.
/// </summary>
private int ReadWithRingBuffer(byte[] buffer, int offset, int count)
{
@@ -392,16 +373,27 @@ internal partial class SharpCompressStream : Stream, IStreamStack
while (count > 0 && _logicalPosition < streamPosition)
{
long bytesFromEnd = streamPosition - _logicalPosition;
int available = _ringBuffer!.ReadFromEnd(bytesFromEnd, buffer, offset, count);
// Verify data is available in ring buffer
if (!_ringBuffer!.CanReadFromEnd(bytesFromEnd))
{
throw new InvalidOperationException(
$"Ring buffer underflow: trying to read {bytesFromEnd} bytes back, "
+ $"but buffer only holds {_ringBuffer.Length} bytes."
);
}
int available = _ringBuffer.ReadFromEnd(bytesFromEnd, buffer, offset, count);
totalRead += available;
offset += available;
count -= available;
_logicalPosition += available;
}
// If more data needed, read from underlying stream
if (count > 0)
// If more data needed and we're caught up, read from underlying stream
if (count > 0 && _logicalPosition == streamPosition)
{
// Use async read if stream doesn't support sync reads (e.g., AsyncOnlyStream)
int read = stream.Read(buffer, offset, count);
if (read > 0)
{

5
src/SharpCompress/Readers/ReaderFactory.Async.cs
View File

@@ -54,7 +54,10 @@ public static partial class ReaderFactory
stream.NotNull(nameof(stream));
options ??= new ReaderOptions() { LeaveStreamOpen = false };
var sharpCompressStream = new SharpCompressStream(stream);
var sharpCompressStream = SharpCompressStream.EnsureSeekable(
stream,
options.RewindableBufferSize
);
sharpCompressStream.StartRecording();
var factories = Factory.Factories.OfType<Factory>();

5
src/SharpCompress/Readers/ReaderFactory.cs
View File

@@ -34,7 +34,10 @@ public static partial class ReaderFactory
stream.NotNull(nameof(stream));
options ??= new ReaderOptions() { LeaveStreamOpen = false };
var sharpCompressStream = SharpCompressStream.EnsureSeekable(stream);
var sharpCompressStream = SharpCompressStream.EnsureSeekable(
stream,
options.RewindableBufferSize
);
sharpCompressStream.StartRecording();
var factories = Factories.Factory.Factories.OfType<Factories.Factory>();

43
src/SharpCompress/Readers/ReaderOptions.cs
View File

@@ -36,4 +36,47 @@ public class ReaderOptions : OptionsBase
/// When set, progress updates will be reported as entries are extracted.
/// </summary>
public IProgress<ProgressReport>? Progress { get; set; }
/// <summary>
/// Size of the rewindable buffer for non-seekable streams.
/// Used during format detection to enable multiple rewinds.
/// </summary>
/// <remarks>
/// <para>
/// When opening archives from non-seekable streams (network streams, pipes,
/// compressed streams), SharpCompress uses a ring buffer to enable format
/// auto-detection. This buffer allows the library to try multiple decoders
/// by rewinding and re-reading the same data.
/// </para>
/// <para>
/// <b>Default:</b> Constants.RewindableBufferSize (81920 bytes / 81KB)
/// </para>
/// <para>
/// <b>Typical usage:</b> 500-1000 bytes for most archives
/// </para>
/// <para>
/// <b>Increase if:</b>
/// <list type="bullet">
/// <item>Opening self-extracting RAR archives (may need 512KB+)</item>
/// <item>Format detection fails with "recording anchor" errors</item>
/// <item>Using custom formats with large headers</item>
/// </list>
/// </para>
/// <para>
/// <b>Memory impact:</b> Buffer is allocated for non-seekable streams only.
/// Seekable streams (FileStream, MemoryStream) use zero-copy seeking instead.
/// </para>
/// </remarks>
/// <example>
/// <code>
/// // For self-extracting archives, use larger buffer
/// var options = new ReaderOptions
/// {
/// RewindableBufferSize = 1_048_576, // 1MB
/// LookForHeader = true
/// };
/// using var reader = ReaderFactory.OpenReader(networkStream, options);
/// </code>
/// </example>
public int? RewindableBufferSize { get; set; }
}

5
src/SharpCompress/Readers/Tar/TarReader.cs
View File

@@ -57,7 +57,10 @@ public partial class TarReader : AbstractReader<TarEntry, TarVolume>
{
stream.NotNull(nameof(stream));
options = options ?? new ReaderOptions();
var sharpCompressStream = SharpCompressStream.EnsureSeekable(stream);
var sharpCompressStream = SharpCompressStream.EnsureSeekable(
stream,
options.RewindableBufferSize
);
long pos = sharpCompressStream.Position;
if (GZipArchive.IsGZipFile(sharpCompressStream))
{