Avoid quadratic header parsing when switching archive enumeration APIs

This commit is contained in:
Adam Hathcock
2026-11-13 10:55:55 +00:00
parent 331808d004
commit 4b82e1728e
8 changed files with 258 additions and 47 deletions

View File

@@ -1,5 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Readers;
@@ -51,9 +53,13 @@ public abstract partial class AbstractArchive<TEntry, TVolume>
await _lazyVolumesAsync.EnsureFullyLoaded().ConfigureAwait(false);
}
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast()
private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast(
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
await foreach (var entry in EntriesAsync.ConfigureAwait(false))
await foreach (
var entry in EntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
)
{
yield return entry;
}

View File

@@ -21,12 +21,15 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
[EnumeratorCancellation] CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
await foreach (
var entry in OldEntriesAsync.WithCancellation(cancellationToken).ConfigureAwait(false)
)
{
cancellationToken.ThrowIfCancellationRequested();
yield return entry;
}
cancellationToken.ThrowIfCancellationRequested();
foreach (var entry in newEntries)
{
cancellationToken.ThrowIfCancellationRequested();
@@ -47,6 +50,7 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
var index = 0;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
while (index < originalEntries.Count)
{
cancellationToken.ThrowIfCancellationRequested();
@@ -57,28 +61,25 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
yield break;
}
var generation = ++originalEntriesGeneration;
await using var source = LoadEntriesAsync(VolumesAsync)
.GetAsyncEnumerator(cancellationToken);
for (var skipped = 0; skipped < index; skipped++)
// The cached loader outlives individual enumerations and must not
// retain a caller's cancellation token after that enumeration ends.
originalEntriesAsyncSource ??= base.EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
while (await originalEntriesAsyncSource.MoveNextAsync().ConfigureAwait(false))
{
if (!await source.MoveNextAsync().ConfigureAwait(false))
if (originalEntriesAsyncRead++ < originalEntries.Count)
{
throw new IncompleteArchiveException("Archive entries changed while loading.");
cancellationToken.ThrowIfCancellationRequested();
continue;
}
originalEntries.Add(originalEntriesAsyncSource.Current);
break;
}
while (
generation == originalEntriesGeneration
&& await source.MoveNextAsync().ConfigureAwait(false)
)
{
cancellationToken.ThrowIfCancellationRequested();
originalEntries.Add(source.Current);
index++;
yield return source.Current;
}
if (generation == originalEntriesGeneration)
if (index == originalEntries.Count)
{
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
originalEntriesAsyncSource = null;
originalEntriesLoaded = true;
}
}
@@ -198,6 +199,11 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
public override async ValueTask DisposeAsync()
{
await base.DisposeAsync().ConfigureAwait(false);
originalEntriesSource?.Dispose();
if (originalEntriesAsyncSource is not null)
{
await originalEntriesAsyncSource.DisposeAsync().ConfigureAwait(false);
}
originalEntries.Cast<Entry>().ForEach(x => x.Close());
newEntries.Cast<Entry>().ForEach(x => x.Close());
removedEntries.Cast<Entry>().ForEach(x => x.Close());

View File

@@ -44,7 +44,10 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
private readonly List<TEntry> originalEntries = new();
private readonly LazyReadOnlyCollection<TEntry> originalEntriesView;
private bool originalEntriesLoaded;
private int originalEntriesGeneration;
private IEnumerator<TEntry>? originalEntriesSource;
private IAsyncEnumerator<TEntry>? originalEntriesAsyncSource;
private int originalEntriesRead;
private int originalEntriesAsyncRead;
private readonly List<TEntry> modifiedEntries = new();
private bool hasModifications;
@@ -109,25 +112,23 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
yield break;
}
// Both APIs share entry identities. Switching loaders must restart
// parsing: the other loader may have moved the archive's source stream.
var generation = ++originalEntriesGeneration;
using var source = LoadEntries(Volumes).GetEnumerator();
for (var skipped = 0; skipped < index; skipped++)
// Resume the base cache instead of reparsing its prefix on every API
// switch. Each loader scans at most once; only the first entry object
// seen at each index is exposed to callers and removal checks.
originalEntriesSource ??= base.Entries.GetEnumerator();
while (originalEntriesSource.MoveNext())
{
if (!source.MoveNext())
if (originalEntriesRead++ < originalEntries.Count)
{
throw new IncompleteArchiveException("Archive entries changed while loading.");
continue;
}
originalEntries.Add(originalEntriesSource.Current);
break;
}
while (generation == originalEntriesGeneration && source.MoveNext())
{
originalEntries.Add(source.Current);
index++;
yield return source.Current;
}
if (generation == originalEntriesGeneration)
if (index == originalEntries.Count)
{
originalEntriesSource.Dispose();
originalEntriesSource = null;
originalEntriesLoaded = true;
}
}
@@ -306,6 +307,7 @@ public abstract partial class AbstractWritableArchive<TEntry, TVolume, TOptions>
public override void Dispose()
{
base.Dispose();
originalEntriesSource?.Dispose();
originalEntries.Cast<Entry>().ForEach(x => x.Close());
newEntries.Cast<Entry>().ForEach(x => x.Close());
removedEntries.Cast<Entry>().ForEach(x => x.Close());

View File

@@ -21,12 +21,17 @@ internal static partial class TarHeaderFactory
#endif
var globalPaxMetadata = new TarHeader.PaxMetadata();
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
while (true)
{
TarHeader? header = null;
try
{
if (mode == StreamingMode.Seekable)
{
stream.Position = position;
}
header = new TarHeader(archiveEncoding);
if (!await header.ReadAsync(reader, globalPaxMetadata).ConfigureAwait(false))
{
@@ -42,6 +47,7 @@ internal static partial class TarHeaderFactory
// Reject padded sizes or end offsets that would wrap into a backward seek.
stream.Position = checked(stream.Position + paddedSize);
position = stream.Position;
}
break;
case StreamingMode.Streaming:

View File

@@ -14,11 +14,17 @@ internal static partial class TarHeaderFactory
)
{
var globalPaxMetadata = new TarHeader.PaxMetadata();
var position = mode == StreamingMode.Seekable ? stream.Position : 0;
while (true)
{
TarHeader? header = null;
try
{
if (mode == StreamingMode.Seekable)
{
// Another entry reader or loader may have moved the shared stream.
stream.Position = position;
}
var reader = new BinaryReader(stream, archiveEncoding.Default, leaveOpen: false);
header = new TarHeader(archiveEncoding);
@@ -36,6 +42,7 @@ internal static partial class TarHeaderFactory
// Reject padded sizes or end offsets that would wrap into a backward seek.
stream.Position = checked(stream.Position + paddedSize);
position = stream.Position;
}
break;
case StreamingMode.Streaming:

View File

@@ -149,4 +149,25 @@ public class GZipArchiveCacheTests : TestBase
}
Assert.True(input.CanRead);
}
[Fact]
public async Task Cancelling_An_Enumeration_Does_Not_Poison_The_Shared_Cache()
{
using var input = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"));
await using var archive = await GZipArchive.OpenAsyncArchive(
input,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
using var cancellation = new CancellationTokenSource();
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
{
Assert.True(await entries.MoveNextAsync());
cancellation.Cancel();
await Assert.ThrowsAnyAsync<System.OperationCanceledException>(async () =>
await entries.MoveNextAsync()
);
}
Assert.Single(await archive.EntriesAsync.ToListAsync());
}
}

View File

@@ -399,14 +399,12 @@ public class SevenZipHeaderAllocationTests
if (minorVersion > 2)
{
await Assert.ThrowsAnyAsync<SharpCompressException>(async () =>
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
await archive.EntriesAsync.ToListAsync()
);
}
else
{
var entry = Assert.Single(
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
);
var entry = Assert.Single(await archive.EntriesAsync.ToListAsync());
Assert.True(entry.IsDirectory);
Assert.Equal("x", entry.Key);
}
@@ -444,7 +442,7 @@ public class SevenZipHeaderAllocationTests
cancellationToken: CancellationToken.None
);
await Assert.ThrowsAsync<IncompleteArchiveException>(async () =>
await archive.EntriesAsync.ToListAsync(CancellationToken.None)
await archive.EntriesAsync.ToListAsync()
);
}
else

View File

@@ -38,11 +38,11 @@ public class WritableArchiveIdentityTests
var asyncWritable = (IWritableAsyncArchive)archive;
// Leave the first loader partially consumed before switching APIs.
var entry = asyncFirst
? await asyncWritable.EntriesAsync.FirstAsync(CancellationToken.None)
? await asyncWritable.EntriesAsync.FirstAsync()
: archive.Entries.First();
var otherEntry = asyncFirst
? archive.Entries.First()
: await asyncWritable.EntriesAsync.FirstAsync(CancellationToken.None);
: await asyncWritable.EntriesAsync.FirstAsync();
Assert.Same(entry, otherEntry);
using var source = new MemoryStream(new byte[] { 42 });
@@ -65,7 +65,7 @@ public class WritableArchiveIdentityTests
}
}
var syncEntries = archive.Entries.ToList();
var asyncEntries = await asyncWritable.EntriesAsync.ToListAsync(CancellationToken.None);
var asyncEntries = await asyncWritable.EntriesAsync.ToListAsync();
Assert.Equal(new[] { "keep", "new" }, syncEntries.Select(e => e.Key));
Assert.Equal(syncEntries, asyncEntries);
@@ -96,7 +96,7 @@ public class WritableArchiveIdentityTests
var asyncArchive = (IWritableAsyncArchive)archive;
if (asyncFirst)
{
var entry = await asyncArchive.EntriesAsync.FirstAsync(CancellationToken.None);
var entry = await asyncArchive.EntriesAsync.FirstAsync();
((IWritableArchive)archive).RemoveEntry(entry);
}
else
@@ -104,10 +104,7 @@ public class WritableArchiveIdentityTests
await asyncArchive.RemoveEntryAsync(archive.Entries.First());
}
var remaining = Assert.Single(archive.Entries);
Assert.Same(
remaining,
Assert.Single(await asyncArchive.EntriesAsync.ToListAsync(CancellationToken.None))
);
Assert.Same(remaining, Assert.Single(await asyncArchive.EntriesAsync.ToListAsync()));
using var payload = remaining.OpenEntryStream();
Assert.Equal(2, payload.ReadByte());
}
@@ -137,6 +134,174 @@ public class WritableArchiveIdentityTests
Assert.False(await asyncEntries.MoveNextAsync());
}
[Theory]
[InlineData(ArchiveType.Zip)]
[InlineData(ArchiveType.Tar)]
public async Task Alternating_Loaders_Read_Only_A_Linear_Number_Of_Headers(ArchiveType type)
{
const int count = 128;
var names = Enumerable.Range(0, count).Select(i => $"entry-{i}").ToArray();
using var data = CreateArchive(type, names);
using var input = new CountingReadStream(data);
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
using var syncEntries = archive.Entries.GetEnumerator();
await using var asyncEntries = ((IAsyncArchive)archive).EntriesAsync.GetAsyncEnumerator(
CancellationToken.None
);
for (var i = 0; i < count; i++)
{
if ((i & 1) == 0)
{
Assert.True(syncEntries.MoveNext());
Assert.True(await asyncEntries.MoveNextAsync());
}
else
{
Assert.True(await asyncEntries.MoveNextAsync());
Assert.True(syncEntries.MoveNext());
}
Assert.Same(syncEntries.Current, asyncEntries.Current);
Assert.Equal(names[i], syncEntries.Current.Key);
}
Assert.False(syncEntries.MoveNext());
Assert.False(await asyncEntries.MoveNextAsync());
// Allow two scans and ZIP's end-of-directory search, but not a prefix
// reparse for every entry. This bound does not depend on timing.
Assert.True(
input.BytesRead <= input.Length * 6,
$"Read {input.BytesRead} bytes for a {input.Length}-byte archive."
);
}
[Theory]
[InlineData(ArchiveType.Zip, false)]
[InlineData(ArchiveType.Zip, true)]
[InlineData(ArchiveType.Tar, false)]
[InlineData(ArchiveType.Tar, true)]
public async Task Partial_Enumerations_Stay_Lazy_And_Reuse_Their_Loader(
ArchiveType type,
bool useAsync
)
{
var names = Enumerable.Range(0, 16).Select(i => $"entry-{i}").ToArray();
using var data = CreateArchive(type, names);
using var input = new CountingReadStream(data);
using var archive = ArchiveFactory.OpenArchive(input, ReaderOptions.ForExternalStream);
var asyncArchive = (IAsyncArchive)archive;
var first = useAsync
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
var readsAfterFirst = input.BytesRead;
var repeated = useAsync
? await asyncArchive.EntriesAsync.FirstAsync()
: archive.Entries.First();
Assert.Same(first, repeated);
Assert.Equal(readsAfterFirst, input.BytesRead);
var entries = useAsync
? await asyncArchive.EntriesAsync.ToListAsync()
: archive.Entries.ToList();
Assert.Equal(names, entries.Select(e => e.Key));
Assert.True(
input.BytesRead > readsAfterFirst,
"The first request must not load every header."
);
}
[Theory]
[InlineData(ArchiveType.Zip)]
[InlineData(ArchiveType.Tar)]
public async Task Cancellation_During_A_Read_Does_Not_Lose_An_Entry(ArchiveType type)
{
using var data = CreateArchive(type, "first", "second", "third");
using var input = new CountingReadStream(data);
using var cancellation = new CancellationTokenSource();
await using var archive = await ArchiveFactory.OpenAsyncArchive(
input,
ReaderOptions.ForExternalStream,
CancellationToken.None
);
var first = await archive.EntriesAsync.FirstAsync();
await using (var entries = archive.EntriesAsync.GetAsyncEnumerator(cancellation.Token))
{
Assert.True(await entries.MoveNextAsync());
Assert.Same(first, entries.Current);
input.AfterRead = cancellation.Cancel;
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () =>
await entries.MoveNextAsync()
);
}
input.AfterRead = null;
var remaining = await archive.EntriesAsync.ToListAsync();
Assert.Equal(new[] { "first", "second", "third" }, remaining.Select(e => e.Key));
Assert.Same(first, remaining[0]);
}
private sealed class CountingReadStream(Stream source) : Stream
{
public long BytesRead { get; private set; }
public Action? AfterRead { get; set; }
public override bool CanRead => source.CanRead;
public override bool CanSeek => source.CanSeek;
public override bool CanWrite => false;
public override long Length => source.Length;
public override long Position
{
get => source.Position;
set => source.Position = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
var read = source.Read(buffer, offset, count);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
public override async Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
var read = await source.ReadAsync(buffer, offset, count, cancellationToken);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
#if !LEGACY_DOTNET
public override int Read(Span<byte> buffer)
{
var read = source.Read(buffer);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
var read = await source.ReadAsync(buffer, cancellationToken);
BytesRead += read;
AfterRead?.Invoke();
return read;
}
#endif
public override long Seek(long offset, SeekOrigin origin) => source.Seek(offset, origin);
public override void Flush() => source.Flush();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
}
private static MemoryStream CreateArchive(ArchiveType type, params string[] names)
{
var input = new MemoryStream();