diff --git a/src/SharpCompress/Factories/GZipFactory.cs b/src/SharpCompress/Factories/GZipFactory.cs index bda6ef1f..4649f0fc 100644 --- a/src/SharpCompress/Factories/GZipFactory.cs +++ b/src/SharpCompress/Factories/GZipFactory.cs @@ -151,13 +151,16 @@ public class GZipFactory CompressionContext.FromStream(sharpCompressStream).WithReaderOptions(options) ) ); - if (TarArchive.IsTarFile(testStream)) + var isTarArchive = TarArchive.IsTarFile(testStream); + + // The TAR probe can consume arbitrary compressed input before it rejects a stream. + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - sharpCompressStream.StopRecording(); reader = new TarReader(sharpCompressStream, options, CompressionType.GZip); return true; } - sharpCompressStream.StopRecording(); reader = OpenReader(sharpCompressStream, options); return true; } @@ -185,12 +188,13 @@ public class GZipFactory var tarReader = await new TarFactory() .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); if (tarReader is not null) { return tarReader; } - sharpCompressStream.StopRecording(); return await OpenAsyncReader(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Factories/LzwFactory.cs b/src/SharpCompress/Factories/LzwFactory.cs index bd5569d1..da475d8a 100644 --- a/src/SharpCompress/Factories/LzwFactory.cs +++ b/src/SharpCompress/Factories/LzwFactory.cs @@ -65,14 +65,17 @@ public class LzwFactory : Factory, IReaderFactory ) ) { - if (TarArchive.IsTarFile(testStream)) + var isTarArchive = TarArchive.IsTarFile(testStream); + + // The TAR probe can consume arbitrary compressed input before it rejects a stream. + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); + if (isTarArchive) { - sharpCompressStream.StopRecording(); reader = new TarReader(sharpCompressStream, options, CompressionType.Lzw); return true; } } - sharpCompressStream.StopRecording(); reader = OpenReader(sharpCompressStream, options); return true; } @@ -100,12 +103,13 @@ public class LzwFactory : Factory, IReaderFactory var tarReader = await new TarFactory() .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); + sharpCompressStream.Rewind(); + sharpCompressStream.StopRecording(); if (tarReader is not null) { return tarReader; } - sharpCompressStream.StopRecording(); return await OpenAsyncReader(sharpCompressStream, options, cancellationToken) .ConfigureAwait(false); } diff --git a/tests/SharpCompress.Test/LargeArchiveTests.cs b/tests/SharpCompress.Test/LargeArchiveTests.cs new file mode 100644 index 00000000..fc94c935 --- /dev/null +++ b/tests/SharpCompress.Test/LargeArchiveTests.cs @@ -0,0 +1,187 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Crypto; +using SharpCompress.Readers; +using SharpCompress.Test.Mocks; +using Xunit; + +namespace SharpCompress.Test; + +[Collection(LargeArchiveCollection.Name)] +public class LargeArchiveTests : TestBase +{ + private const long LargeFileSize = 64L * 1024 * 1024; + private const uint LargeFileCrc = 0xF9081EB0; + private const int BufferSize = 64 * 1024; + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.7z")] + public void OpenArchive_ShouldStreamLargeEntry(string fixtureName) + { + using var stream = File.OpenRead(GetArchiveFixturePath(fixtureName)); + using var archive = ArchiveFactory.OpenArchive(stream); + + VerifyArchive(archive); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.7z")] + public async Task OpenAsyncArchive_ShouldStreamLargeEntry(string fixtureName) + { + await using var stream = new AsyncOnlyStream( + File.OpenRead(await GetArchiveFixturePathAsync(fixtureName)) + ); + await using var archive = await ArchiveFactory.OpenAsyncArchive(stream); + + var entry = await GetSingleEntryAsync(archive); + await using var entryStream = await entry.OpenEntryStreamAsync(); + await VerifyContentAsync(entry.Key, entryStream); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.tar.gz")] + public void OpenReader_ShouldStreamLargeEntry(string fixtureName) + { + using var stream = File.OpenRead(GetFixturePath(fixtureName)); + using var reader = ReaderFactory.OpenReader(stream); + + VerifyReader(reader); + } + + [Theory] + [InlineData("Large/Large.zip")] + [InlineData("Large/Large.tar")] + [InlineData("Large/Large.gz")] + [InlineData("Large/Large.rar")] + [InlineData("Large/Large.tar.gz")] + public async Task OpenAsyncReader_ShouldStreamLargeEntry(string fixtureName) + { + await using var stream = new AsyncOnlyStream(File.OpenRead(GetFixturePath(fixtureName))); + await using var reader = await ReaderFactory.OpenAsyncReader(stream); + + Assert.True(await reader.MoveToNextEntryAsync()); + Assert.False(reader.Entry.IsDirectory); + await using var entryStream = await reader.OpenEntryStreamAsync(); + await VerifyContentAsync(reader.Entry.Key, entryStream); + Assert.False(await reader.MoveToNextEntryAsync()); + } + + private static string GetFixturePath(string fixtureName) => + Path.Combine(TEST_ARCHIVES_PATH, fixtureName); + + private string GetArchiveFixturePath(string fixtureName) => + fixtureName == "Large/Large.tar" ? MaterializeTarFixture() : GetFixturePath(fixtureName); + + private async Task GetArchiveFixturePathAsync(string fixtureName) => + fixtureName == "Large/Large.tar" + ? await MaterializeTarFixtureAsync() + : GetFixturePath(fixtureName); + + private string MaterializeTarFixture() + { + var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar"); + using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz")); + using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress); + using var tarStream = File.Create(tarPath); + gzipStream.CopyTo(tarStream); + return tarPath; + } + + private async Task MaterializeTarFixtureAsync() + { + var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar"); + using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz")); + using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress); + using var tarStream = File.Create(tarPath); + await gzipStream.CopyToAsync(tarStream); + return tarPath; + } + + private static void VerifyArchive(IArchive archive) + { + var entry = Assert.Single(archive.Entries); + + Assert.False(entry.IsDirectory); + using var entryStream = entry.OpenEntryStream(); + VerifyContent(entry.Key, entryStream); + } + + private static async Task GetSingleEntryAsync(IAsyncArchive archive) + { + IArchiveEntry? entry = null; + await foreach (var candidate in archive.EntriesAsync) + { + Assert.Null(entry); + entry = candidate; + } + + return entry ?? throw new InvalidOperationException("The archive contains no entries."); + } + + private static void VerifyReader(IReader reader) + { + Assert.True(reader.MoveToNextEntry()); + Assert.False(reader.Entry.IsDirectory); + using var entryStream = reader.OpenEntryStream(); + VerifyContent(reader.Entry.Key, entryStream); + Assert.False(reader.MoveToNextEntry()); + } + + private static void VerifyContent(string? key, Stream entryStream) + { + Assert.Equal("large.bin", key); + + using var crcStream = new Crc32Stream(Stream.Null); + var buffer = new byte[BufferSize]; + long length = 0; + int bytesRead; + while ((bytesRead = entryStream.Read(buffer, 0, buffer.Length)) > 0) + { + crcStream.Write(buffer, 0, bytesRead); + length += bytesRead; + } + + Assert.Equal(LargeFileSize, length); + Assert.Equal(LargeFileCrc, crcStream.Crc); + } + + private static async Task VerifyContentAsync(string? key, Stream entryStream) + { + Assert.Equal("large.bin", key); + + using var crcStream = new Crc32Stream(Stream.Null); + var buffer = new byte[BufferSize]; + long length = 0; + int bytesRead; + while ((bytesRead = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + await crcStream.WriteAsync(buffer, 0, bytesRead); + length += bytesRead; + } + + Assert.Equal(LargeFileSize, length); + Assert.Equal(LargeFileCrc, crcStream.Crc); + } +} + +[CollectionDefinition(LargeArchiveCollection.Name, DisableParallelization = true)] +public sealed class LargeArchiveCollection +{ + public const string Name = "Large archive fixtures"; +} diff --git a/tests/TestArchives/Archives/Large/Large.7z b/tests/TestArchives/Archives/Large/Large.7z new file mode 100644 index 00000000..719ea18d Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.7z differ diff --git a/tests/TestArchives/Archives/Large/Large.gz b/tests/TestArchives/Archives/Large/Large.gz new file mode 100644 index 00000000..7401121e Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.gz differ diff --git a/tests/TestArchives/Archives/Large/Large.rar b/tests/TestArchives/Archives/Large/Large.rar new file mode 100644 index 00000000..6f0a65d4 Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.rar differ diff --git a/tests/TestArchives/Archives/Large/Large.tar.gz b/tests/TestArchives/Archives/Large/Large.tar.gz new file mode 100644 index 00000000..007fef08 Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.tar.gz differ diff --git a/tests/TestArchives/Archives/Large/Large.zip b/tests/TestArchives/Archives/Large/Large.zip new file mode 100644 index 00000000..5b07336f Binary files /dev/null and b/tests/TestArchives/Archives/Large/Large.zip differ diff --git a/tests/TestArchives/Archives/Large/README.md b/tests/TestArchives/Archives/Large/README.md new file mode 100644 index 00000000..4992f31b --- /dev/null +++ b/tests/TestArchives/Archives/Large/README.md @@ -0,0 +1,38 @@ +# Large Test Archives + +Each fixture contains one `large.bin` entry with a 67,108,864-byte (64 MiB) +deterministic, repeated text pattern with a 2 KiB `0xFF` prefix. The entry CRC-32 is +`f9081eb0`. + +| Fixture | Archive API | Reader API | +| --- | --- | --- | +| `Large.zip` | Yes | Yes | +| Generated `Large.tar` | Yes | N/A | +| `Large.gz` | Yes | Yes | +| `Large.rar` | Yes | Yes | +| `Large.7z` | Yes | No | +| `Large.tar.gz` | No | Yes | + +The compressible payload keeps the compressed fixtures small while requiring a full +64 MiB decompression to validate each API. The Archive API test expands `Large.tar.gz` +to a scratch `Large.tar` before exercising raw TAR support, so the 64 MiB TAR file is +not committed. + +## Regenerating + +The fixtures were created with `zip`, `tar`, `gzip`, RAR 7.22, and 7-Zip. Run these +commands from a temporary directory after replacing `` with the repository root: + +```sh +yes "SharpCompress large fixture" | head -c 67108864 > large.bin +printf '\377%.0s' {1..2048} > prefix.bin +dd if=prefix.bin of=large.bin bs=2048 count=1 conv=notrunc +touch -t 202001010000 large.bin +mkdir -p /tests/TestArchives/Archives/Large +zip -X -9 -j /tests/TestArchives/Archives/Large/Large.zip large.bin +COPYFILE_DISABLE=1 tar -cf large.tar large.bin +gzip -9 -c large.bin > /tests/TestArchives/Archives/Large/Large.gz +gzip -n -9 -c large.tar > /tests/TestArchives/Archives/Large/Large.tar.gz +rar a -ma5 -m5 -ep /tests/TestArchives/Archives/Large/Large.rar large.bin +7z a -t7z -mx=9 /tests/TestArchives/Archives/Large/Large.7z large.bin +```