mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-11-18 18:32:34 +00:00
Avoid quadratic header parsing when switching archive enumeration APIs
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Readers;
|
||||
@@ -51,9 +53,13 @@ public abstract partial class AbstractArchive<TEntry, TVolume>
|
||||
await _lazyVolumesAsync.EnsureFullyLoaded().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast()
|
||||
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
await foreach (var entry in EntriesAsync.ConfigureAwait(false))
|
||||
await foreach (
|
||||
var entry in EntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
|
||||
)
|
||||
{
|
||||
yield return entry;
|
||||
}
|
||||
|
||||
@@ -21,12 +21,15 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await foreach (
|
||||
var entry in OldEntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
|
||||
)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
yield return entry;
|
||||
}
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
foreach (var entry in newEntries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
@@ -47,6 +50,7 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
var index = 0;
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
while (index < originalEntries.Count)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
@@ -57,28 +61,25 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
yield break;
|
||||
}
|
||||
|
||||
var generation = ++originalEntriesGeneration;
|
||||
await using var source = LoadEntriesAsync(VolumesAsync)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
for (var skipped = 0; skipped < index; skipped++)
|
||||
// The cached loader outlives individual enumerations and must not
|
||||
// retain a caller's cancellation token after that enumeration ends.
|
||||
originalEntriesAsyncSource ??= base.EntriesAsync.GetAsyncEnumerator(
|
||||
CancellationToken.None
|
||||
);
|
||||
while (await originalEntriesAsyncSource.MoveNextAsync().ConfigureAwait(false))
|
||||
{
|
||||
if (!await source.MoveNextAsync().ConfigureAwait(false))
|
||||
if (originalEntriesAsyncRead++ < originalEntries.Count)
|
||||
{
|
||||
throw new IncompleteArchiveException("Archive entries changed while loading.");
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
continue;
|
||||
}
|
||||
originalEntries.Add(originalEntriesAsyncSource.Current);
|
||||
break;
|
||||
}
|
||||
while (
|
||||
generation == originalEntriesGeneration
|
||||
&& await source.MoveNextAsync().ConfigureAwait(false)
|
||||
)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
originalEntries.Add(source.Current);
|
||||
index++;
|
||||
yield return source.Current;
|
||||
}
|
||||
if (generation == originalEntriesGeneration)
|
||||
if (index == originalEntries.Count)
|
||||
{
|
||||
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
|
||||
originalEntriesAsyncSource = null;
|
||||
originalEntriesLoaded = true;
|
||||
}
|
||||
}
|
||||
@@ -198,6 +199,11 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
public override async ValueTask DisposeAsync()
|
||||
{
|
||||
await base.DisposeAsync().ConfigureAwait(false);
|
||||
originalEntriesSource?.Dispose();
|
||||
if (originalEntriesAsyncSource is not null)
|
||||
{
|
||||
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
originalEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
newEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
removedEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
|
||||
@@ -44,7 +44,10 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
private readonly List<TEntry> originalEntries = new();
|
||||
private readonly LazyReadOnlyCollection<TEntry> originalEntriesView;
|
||||
private bool originalEntriesLoaded;
|
||||
private int originalEntriesGeneration;
|
||||
private IEnumerator<TEntry>? originalEntriesSource;
|
||||
private IAsyncEnumerator<TEntry>? originalEntriesAsyncSource;
|
||||
private int originalEntriesRead;
|
||||
private int originalEntriesAsyncRead;
|
||||
|
||||
private readonly List<TEntry> modifiedEntries = new();
|
||||
private bool hasModifications;
|
||||
@@ -109,25 +112,23 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Both APIs share entry identities. Switching loaders must restart
|
||||
// parsing: the other loader may have moved the archive's source stream.
|
||||
var generation = ++originalEntriesGeneration;
|
||||
using var source = LoadEntries(Volumes).GetEnumerator();
|
||||
for (var skipped = 0; skipped < index; skipped++)
|
||||
// Resume the base cache instead of reparsing its prefix on every API
|
||||
// switch. Each loader scans at most once; only the first entry object
|
||||
// seen at each index is exposed to callers and removal checks.
|
||||
originalEntriesSource ??= base.Entries.GetEnumerator();
|
||||
while (originalEntriesSource.MoveNext())
|
||||
{
|
||||
if (!source.MoveNext())
|
||||
if (originalEntriesRead++ < originalEntries.Count)
|
||||
{
|
||||
throw new IncompleteArchiveException("Archive entries changed while loading.");
|
||||
continue;
|
||||
}
|
||||
originalEntries.Add(originalEntriesSource.Current);
|
||||
break;
|
||||
}
|
||||
while (generation == originalEntriesGeneration && source.MoveNext())
|
||||
{
|
||||
originalEntries.Add(source.Current);
|
||||
index++;
|
||||
yield return source.Current;
|
||||
}
|
||||
if (generation == originalEntriesGeneration)
|
||||
if (index == originalEntries.Count)
|
||||
{
|
||||
originalEntriesSource.Dispose();
|
||||
originalEntriesSource = null;
|
||||
originalEntriesLoaded = true;
|
||||
}
|
||||
}
|
||||
@@ -306,6 +307,7 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
|
||||
public override void Dispose()
|
||||
{
|
||||
base.Dispose();
|
||||
originalEntriesSource?.Dispose();
|
||||
originalEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
newEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
removedEntries.Cast<Entry>().ForEach(x => x.Close());
|
||||
|
||||
@@ -21,12 +21,17 @@ internal static partial class TarHeaderFactory
|
||||
#endif
|
||||
|
||||
var globalPaxMetadata = new TarHeader.PaxMetadata();
|
||||
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
TarHeader? header = null;
|
||||
try
|
||||
{
|
||||
if (mode == StreamingMode.Seekable)
|
||||
{
|
||||
stream.Position = position;
|
||||
}
|
||||
header = new TarHeader(archiveEncoding);
|
||||
if (!await header.ReadAsync(reader, globalPaxMetadata).ConfigureAwait(false))
|
||||
{
|
||||
@@ -42,6 +47,7 @@ internal static partial class TarHeaderFactory
|
||||
|
||||
// Reject padded sizes or end offsets that would wrap into a backward seek.
|
||||
stream.Position = checked(stream.Position + paddedSize);
|
||||
position = stream.Position;
|
||||
}
|
||||
break;
|
||||
case StreamingMode.Streaming:
|
||||
|
||||
@@ -14,11 +14,17 @@ internal static partial class TarHeaderFactory
|
||||
)
|
||||
{
|
||||
var globalPaxMetadata = new TarHeader.PaxMetadata();
|
||||
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
|
||||
while (true)
|
||||
{
|
||||
TarHeader? header = null;
|
||||
try
|
||||
{
|
||||
if (mode == StreamingMode.Seekable)
|
||||
{
|
||||
// Another entry reader or loader may have moved the shared stream.
|
||||
stream.Position = position;
|
||||
}
|
||||
var reader = new BinaryReader(stream, archiveEncoding.Default, leaveOpen: false);
|
||||
header = new TarHeader(archiveEncoding);
|
||||
|
||||
@@ -36,6 +42,7 @@ internal static partial class TarHeaderFactory
|
||||
|
||||
// Reject padded sizes or end offsets that would wrap into a backward seek.
|
||||
stream.Position = checked(stream.Position + paddedSize);
|
||||
position = stream.Position;
|
||||
}
|
||||
break;
|
||||
case StreamingMode.Streaming:
|
||||
|
||||
@@ -149,4 +149,25 @@ public class GZipArchiveCacheTests : TestBase
|
||||
}
|
||||
Assert.True(input.CanRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancelling_An_Enumeration_Does_Not_Poison_The_Shared_Cache()
|
||||
{
|
||||
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
|
||||
await using var archive = await GZipArchive.OpenAsyncArchive(
|
||||
input,
|
||||
ReaderOptions.ForExternalStream,
|
||||
CancellationToken.None
|
||||
);
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
|
||||
{
|
||||
Assert.True(await entries.MoveNextAsync());
|
||||
cancellation.Cancel();
|
||||
await Assert.ThrowsAnyAsync<System.OperationCanceledException>(async () =>
|
||||
await entries.MoveNextAsync()
|
||||
);
|
||||
}
|
||||
Assert.Single(await archive.EntriesAsync.ToListAsync());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,14 +399,12 @@ public class SevenZipHeaderAllocationTests
|
||||
if (minorVersion > 2)
|
||||
{
|
||||
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
|
||||
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
|
||||
await archive.EntriesAsync.ToListAsync()
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
var entry = Assert.Single(
|
||||
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
|
||||
);
|
||||
var entry = Assert.Single(await archive.EntriesAsync.ToListAsync());
|
||||
Assert.True(entry.IsDirectory);
|
||||
Assert.Equal("x", entry.Key);
|
||||
}
|
||||
@@ -444,7 +442,7 @@ public class SevenZipHeaderAllocationTests
|
||||
cancellationToken: CancellationToken.None
|
||||
);
|
||||
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
|
||||
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
|
||||
await archive.EntriesAsync.ToListAsync()
|
||||
);
|
||||
}
|
||||
else
|
||||
|
||||
@@ -38,11 +38,11 @@ public class WritableArchiveIdentityTests
|
||||
var asyncWritable = (IWritableAsyncArchive)archive;
|
||||
// Leave the first loader partially consumed before switching APIs.
|
||||
var entry = asyncFirst
|
||||
? await asyncWritable.EntriesAsync.FirstAsync(CancellationToken.None)
|
||||
? await asyncWritable.EntriesAsync.FirstAsync()
|
||||
: archive.Entries.First();
|
||||
var otherEntry = asyncFirst
|
||||
? archive.Entries.First()
|
||||
: await asyncWritable.EntriesAsync.FirstAsync(CancellationToken.None);
|
||||
: await asyncWritable.EntriesAsync.FirstAsync();
|
||||
Assert.Same(entry, otherEntry);
|
||||
|
||||
using var source = new MemoryStream(new byte[] { 42 });
|
||||
@@ -65,7 +65,7 @@ public class WritableArchiveIdentityTests
|
||||
}
|
||||
}
|
||||
var syncEntries = archive.Entries.ToList();
|
||||
var asyncEntries = await asyncWritable.EntriesAsync.ToListAsync(CancellationToken.None);
|
||||
var asyncEntries = await asyncWritable.EntriesAsync.ToListAsync();
|
||||
Assert.Equal(new[] { "keep", "new" }, syncEntries.Select(e => e.Key));
|
||||
Assert.Equal(syncEntries, asyncEntries);
|
||||
|
||||
@@ -96,7 +96,7 @@ public class WritableArchiveIdentityTests
|
||||
var asyncArchive = (IWritableAsyncArchive)archive;
|
||||
if (asyncFirst)
|
||||
{
|
||||
var entry = await asyncArchive.EntriesAsync.FirstAsync(CancellationToken.None);
|
||||
var entry = await asyncArchive.EntriesAsync.FirstAsync();
|
||||
((IWritableArchive)archive).RemoveEntry(entry);
|
||||
}
|
||||
else
|
||||
@@ -104,10 +104,7 @@ public class WritableArchiveIdentityTests
|
||||
await asyncArchive.RemoveEntryAsync(archive.Entries.First());
|
||||
}
|
||||
var remaining = Assert.Single(archive.Entries);
|
||||
Assert.Same(
|
||||
remaining,
|
||||
Assert.Single(await asyncArchive.EntriesAsync.ToListAsync(CancellationToken.None))
|
||||
);
|
||||
Assert.Same(remaining, Assert.Single(await asyncArchive.EntriesAsync.ToListAsync()));
|
||||
using var payload = remaining.OpenEntryStream();
|
||||
Assert.Equal(2, payload.ReadByte());
|
||||
}
|
||||
@@ -137,6 +134,174 @@ public class WritableArchiveIdentityTests
|
||||
Assert.False(await asyncEntries.MoveNextAsync());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ArchiveType.Zip)]
|
||||
[InlineData(ArchiveType.Tar)]
|
||||
public async Task Alternating_Loaders_Read_Only_A_Linear_Number_Of_Headers(ArchiveType type)
|
||||
{
|
||||
const int count = 128;
|
||||
var names = Enumerable.Range(0, count).Select(i => $"entry-{i}").ToArray();
|
||||
using var data = CreateArchive(type, names);
|
||||
using var input = new CountingReadStream(data);
|
||||
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
|
||||
using var syncEntries = archive.Entries.GetEnumerator();
|
||||
await using var asyncEntries = ((IAsyncArchive)archive).EntriesAsync.GetAsyncEnumerator(
|
||||
CancellationToken.None
|
||||
);
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
if ((i & 1) == 0)
|
||||
{
|
||||
Assert.True(syncEntries.MoveNext());
|
||||
Assert.True(await asyncEntries.MoveNextAsync());
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.True(await asyncEntries.MoveNextAsync());
|
||||
Assert.True(syncEntries.MoveNext());
|
||||
}
|
||||
Assert.Same(syncEntries.Current, asyncEntries.Current);
|
||||
Assert.Equal(names[i], syncEntries.Current.Key);
|
||||
}
|
||||
Assert.False(syncEntries.MoveNext());
|
||||
Assert.False(await asyncEntries.MoveNextAsync());
|
||||
// Allow two scans and ZIP's end-of-directory search, but not a prefix
|
||||
// reparse for every entry. This bound does not depend on timing.
|
||||
Assert.True(
|
||||
input.BytesRead <= input.Length * 6,
|
||||
$"Read {input.BytesRead} bytes for a {input.Length}-byte archive."
|
||||
);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ArchiveType.Zip, false)]
|
||||
[InlineData(ArchiveType.Zip, true)]
|
||||
[InlineData(ArchiveType.Tar, false)]
|
||||
[InlineData(ArchiveType.Tar, true)]
|
||||
public async Task Partial_Enumerations_Stay_Lazy_And_Reuse_Their_Loader(
|
||||
ArchiveType type,
|
||||
bool useAsync
|
||||
)
|
||||
{
|
||||
var names = Enumerable.Range(0, 16).Select(i => $"entry-{i}").ToArray();
|
||||
using var data = CreateArchive(type, names);
|
||||
using var input = new CountingReadStream(data);
|
||||
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
|
||||
var asyncArchive = (IAsyncArchive)archive;
|
||||
var first = useAsync
|
||||
? await asyncArchive.EntriesAsync.FirstAsync()
|
||||
: archive.Entries.First();
|
||||
var readsAfterFirst = input.BytesRead;
|
||||
var repeated = useAsync
|
||||
? await asyncArchive.EntriesAsync.FirstAsync()
|
||||
: archive.Entries.First();
|
||||
Assert.Same(first, repeated);
|
||||
Assert.Equal(readsAfterFirst, input.BytesRead);
|
||||
var entries = useAsync
|
||||
? await asyncArchive.EntriesAsync.ToListAsync()
|
||||
: archive.Entries.ToList();
|
||||
Assert.Equal(names, entries.Select(e => e.Key));
|
||||
Assert.True(
|
||||
input.BytesRead > readsAfterFirst,
|
||||
"The first request must not load every header."
|
||||
);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ArchiveType.Zip)]
|
||||
[InlineData(ArchiveType.Tar)]
|
||||
public async Task Cancellation_During_A_Read_Does_Not_Lose_An_Entry(ArchiveType type)
|
||||
{
|
||||
using var data = CreateArchive(type, "first", "second", "third");
|
||||
using var input = new CountingReadStream(data);
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
await using var archive = await ArchiveFactory.OpenAsyncArchive(
|
||||
input,
|
||||
ReaderOptions.ForExternalStream,
|
||||
CancellationToken.None
|
||||
);
|
||||
var first = await archive.EntriesAsync.FirstAsync();
|
||||
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
|
||||
{
|
||||
Assert.True(await entries.MoveNextAsync());
|
||||
Assert.Same(first, entries.Current);
|
||||
input.AfterRead = cancellation.Cancel;
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
|
||||
await entries.MoveNextAsync()
|
||||
);
|
||||
}
|
||||
input.AfterRead = null;
|
||||
var remaining = await archive.EntriesAsync.ToListAsync();
|
||||
Assert.Equal(new[] { "first", "second", "third" }, remaining.Select(e => e.Key));
|
||||
Assert.Same(first, remaining[0]);
|
||||
}
|
||||
|
||||
private sealed class CountingReadStream(Stream source) : Stream
|
||||
{
|
||||
public long BytesRead { get; private set; }
|
||||
public Action? AfterRead { get; set; }
|
||||
public override bool CanRead => source.CanRead;
|
||||
public override bool CanSeek => source.CanSeek;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => source.Length;
|
||||
public override long Position
|
||||
{
|
||||
get => source.Position;
|
||||
set => source.Position = value;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
var read = source.Read(buffer, offset, count);
|
||||
BytesRead += read;
|
||||
AfterRead?.Invoke();
|
||||
return read;
|
||||
}
|
||||
|
||||
public override async Task<int> ReadAsync(
|
||||
byte[] buffer,
|
||||
int offset,
|
||||
int count,
|
||||
CancellationToken cancellationToken
|
||||
)
|
||||
{
|
||||
var read = await source.ReadAsync(buffer, offset, count, cancellationToken);
|
||||
BytesRead += read;
|
||||
AfterRead?.Invoke();
|
||||
return read;
|
||||
}
|
||||
|
||||
#if !LEGACY_DOTNET
|
||||
public override int Read(Span<byte> buffer)
|
||||
{
|
||||
var read = source.Read(buffer);
|
||||
BytesRead += read;
|
||||
AfterRead?.Invoke();
|
||||
return read;
|
||||
}
|
||||
|
||||
public override async ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
var read = await source.ReadAsync(buffer, cancellationToken);
|
||||
BytesRead += read;
|
||||
AfterRead?.Invoke();
|
||||
return read;
|
||||
}
|
||||
#endif
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => source.Seek(offset, origin);
|
||||
|
||||
public override void Flush() => source.Flush();
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count) =>
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
private static MemoryStream CreateArchive(ArchiveType type, params string[] names)
|
||||
{
|
||||
var input = new MemoryStream();
|
||||
|
||||
Reference in New Issue
Block a user