add large file tests and GZip/Lzw factory fixes

src/SharpCompress/Factories/GZipFactory.cs

@@ -151,13 +151,16 @@ public class GZipFactory
                     CompressionContext.FromStream(sharpCompressStream).WithReaderOptions(options)
                 )
             );
-            if (TarArchive.IsTarFile(testStream))
+            var isTarArchive = TarArchive.IsTarFile(testStream);
+
+            // The TAR probe can consume arbitrary compressed input before it rejects a stream.
+            sharpCompressStream.Rewind();
+            sharpCompressStream.StopRecording();
+            if (isTarArchive)
             {
-                sharpCompressStream.StopRecording();
                 reader = new TarReader(sharpCompressStream, options, CompressionType.GZip);
                 return true;
             }
-            sharpCompressStream.StopRecording();
             reader = OpenReader(sharpCompressStream, options);
             return true;
         }
@@ -185,12 +188,13 @@ public class GZipFactory
         var tarReader = await new TarFactory()
             .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken)
             .ConfigureAwait(false);
+        sharpCompressStream.Rewind();
+        sharpCompressStream.StopRecording();
         if (tarReader is not null)
         {
             return tarReader;
         }
 
-        sharpCompressStream.StopRecording();
         return await OpenAsyncReader(sharpCompressStream, options, cancellationToken)
             .ConfigureAwait(false);
     }

src/SharpCompress/Factories/LzwFactory.cs

@@ -65,14 +65,17 @@ public class LzwFactory : Factory, IReaderFactory
                 )
             )
             {
-                if (TarArchive.IsTarFile(testStream))
+                var isTarArchive = TarArchive.IsTarFile(testStream);
+
+                // The TAR probe can consume arbitrary compressed input before it rejects a stream.
+                sharpCompressStream.Rewind();
+                sharpCompressStream.StopRecording();
+                if (isTarArchive)
                 {
-                    sharpCompressStream.StopRecording();
                     reader = new TarReader(sharpCompressStream, options, CompressionType.Lzw);
                     return true;
                 }
             }
-            sharpCompressStream.StopRecording();
             reader = OpenReader(sharpCompressStream, options);
             return true;
         }
@@ -100,12 +103,13 @@ public class LzwFactory : Factory, IReaderFactory
         var tarReader = await new TarFactory()
             .TryOpenReaderAsync(sharpCompressStream, options, cancellationToken)
             .ConfigureAwait(false);
+        sharpCompressStream.Rewind();
+        sharpCompressStream.StopRecording();
         if (tarReader is not null)
         {
             return tarReader;
         }
 
-        sharpCompressStream.StopRecording();
         return await OpenAsyncReader(sharpCompressStream, options, cancellationToken)
             .ConfigureAwait(false);
     }

tests/SharpCompress.Test/LargeArchiveTests.cs

@@ -0,0 +1,187 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using SharpCompress.Archives;
+using SharpCompress.Crypto;
+using SharpCompress.Readers;
+using SharpCompress.Test.Mocks;
+using Xunit;
+
+namespace SharpCompress.Test;
+
+[Collection(LargeArchiveCollection.Name)]
+public class LargeArchiveTests : TestBase
+{
+    private const long LargeFileSize = 64L * 1024 * 1024;
+    private const uint LargeFileCrc = 0xF9081EB0;
+    private const int BufferSize = 64 * 1024;
+
+    [Theory]
+    [InlineData("Large/Large.zip")]
+    [InlineData("Large/Large.tar")]
+    [InlineData("Large/Large.gz")]
+    [InlineData("Large/Large.rar")]
+    [InlineData("Large/Large.7z")]
+    public void OpenArchive_ShouldStreamLargeEntry(string fixtureName)
+    {
+        using var stream = File.OpenRead(GetArchiveFixturePath(fixtureName));
+        using var archive = ArchiveFactory.OpenArchive(stream);
+
+        VerifyArchive(archive);
+    }
+
+    [Theory]
+    [InlineData("Large/Large.zip")]
+    [InlineData("Large/Large.tar")]
+    [InlineData("Large/Large.gz")]
+    [InlineData("Large/Large.rar")]
+    [InlineData("Large/Large.7z")]
+    public async Task OpenAsyncArchive_ShouldStreamLargeEntry(string fixtureName)
+    {
+        await using var stream = new AsyncOnlyStream(
+            File.OpenRead(await GetArchiveFixturePathAsync(fixtureName))
+        );
+        await using var archive = await ArchiveFactory.OpenAsyncArchive(stream);
+
+        var entry = await GetSingleEntryAsync(archive);
+        await using var entryStream = await entry.OpenEntryStreamAsync();
+        await VerifyContentAsync(entry.Key, entryStream);
+    }
+
+    [Theory]
+    [InlineData("Large/Large.zip")]
+    [InlineData("Large/Large.tar")]
+    [InlineData("Large/Large.gz")]
+    [InlineData("Large/Large.rar")]
+    [InlineData("Large/Large.tar.gz")]
+    public void OpenReader_ShouldStreamLargeEntry(string fixtureName)
+    {
+        using var stream = File.OpenRead(GetFixturePath(fixtureName));
+        using var reader = ReaderFactory.OpenReader(stream);
+
+        VerifyReader(reader);
+    }
+
+    [Theory]
+    [InlineData("Large/Large.zip")]
+    [InlineData("Large/Large.tar")]
+    [InlineData("Large/Large.gz")]
+    [InlineData("Large/Large.rar")]
+    [InlineData("Large/Large.tar.gz")]
+    public async Task OpenAsyncReader_ShouldStreamLargeEntry(string fixtureName)
+    {
+        await using var stream = new AsyncOnlyStream(File.OpenRead(GetFixturePath(fixtureName)));
+        await using var reader = await ReaderFactory.OpenAsyncReader(stream);
+
+        Assert.True(await reader.MoveToNextEntryAsync());
+        Assert.False(reader.Entry.IsDirectory);
+        await using var entryStream = await reader.OpenEntryStreamAsync();
+        await VerifyContentAsync(reader.Entry.Key, entryStream);
+        Assert.False(await reader.MoveToNextEntryAsync());
+    }
+
+    private static string GetFixturePath(string fixtureName) =>
+        Path.Combine(TEST_ARCHIVES_PATH, fixtureName);
+
+    private string GetArchiveFixturePath(string fixtureName) =>
+        fixtureName == "Large/Large.tar" ? MaterializeTarFixture() : GetFixturePath(fixtureName);
+
+    private async Task<string> GetArchiveFixturePathAsync(string fixtureName) =>
+        fixtureName == "Large/Large.tar"
+            ? await MaterializeTarFixtureAsync()
+            : GetFixturePath(fixtureName);
+
+    private string MaterializeTarFixture()
+    {
+        var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar");
+        using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz"));
+        using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress);
+        using var tarStream = File.Create(tarPath);
+        gzipStream.CopyTo(tarStream);
+        return tarPath;
+    }
+
+    private async Task<string> MaterializeTarFixtureAsync()
+    {
+        var tarPath = Path.Combine(SCRATCH_FILES_PATH, "Large.tar");
+        using var compressedStream = File.OpenRead(GetFixturePath("Large/Large.tar.gz"));
+        using var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress);
+        using var tarStream = File.Create(tarPath);
+        await gzipStream.CopyToAsync(tarStream);
+        return tarPath;
+    }
+
+    private static void VerifyArchive(IArchive archive)
+    {
+        var entry = Assert.Single(archive.Entries);
+
+        Assert.False(entry.IsDirectory);
+        using var entryStream = entry.OpenEntryStream();
+        VerifyContent(entry.Key, entryStream);
+    }
+
+    private static async Task<IArchiveEntry> GetSingleEntryAsync(IAsyncArchive archive)
+    {
+        IArchiveEntry? entry = null;
+        await foreach (var candidate in archive.EntriesAsync)
+        {
+            Assert.Null(entry);
+            entry = candidate;
+        }
+
+        return entry ?? throw new InvalidOperationException("The archive contains no entries.");
+    }
+
+    private static void VerifyReader(IReader reader)
+    {
+        Assert.True(reader.MoveToNextEntry());
+        Assert.False(reader.Entry.IsDirectory);
+        using var entryStream = reader.OpenEntryStream();
+        VerifyContent(reader.Entry.Key, entryStream);
+        Assert.False(reader.MoveToNextEntry());
+    }
+
+    private static void VerifyContent(string? key, Stream entryStream)
+    {
+        Assert.Equal("large.bin", key);
+
+        using var crcStream = new Crc32Stream(Stream.Null);
+        var buffer = new byte[BufferSize];
+        long length = 0;
+        int bytesRead;
+        while ((bytesRead = entryStream.Read(buffer, 0, buffer.Length)) > 0)
+        {
+            crcStream.Write(buffer, 0, bytesRead);
+            length += bytesRead;
+        }
+
+        Assert.Equal(LargeFileSize, length);
+        Assert.Equal(LargeFileCrc, crcStream.Crc);
+    }
+
+    private static async Task VerifyContentAsync(string? key, Stream entryStream)
+    {
+        Assert.Equal("large.bin", key);
+
+        using var crcStream = new Crc32Stream(Stream.Null);
+        var buffer = new byte[BufferSize];
+        long length = 0;
+        int bytesRead;
+        while ((bytesRead = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
+        {
+            await crcStream.WriteAsync(buffer, 0, bytesRead);
+            length += bytesRead;
+        }
+
+        Assert.Equal(LargeFileSize, length);
+        Assert.Equal(LargeFileCrc, crcStream.Crc);
+    }
+}
+
+[CollectionDefinition(LargeArchiveCollection.Name, DisableParallelization = true)]
+public sealed class LargeArchiveCollection
+{
+    public const string Name = "Large archive fixtures";
+}

tests/TestArchives/Archives/Large/README.md

@@ -0,0 +1,38 @@
+# Large Test Archives
+
+Each fixture contains one `large.bin` entry with a 67,108,864-byte (64 MiB)
+deterministic, repeated text pattern with a 2 KiB `0xFF` prefix. The entry CRC-32 is
+`f9081eb0`.
+
+| Fixture | Archive API | Reader API |
+| --- | --- | --- |
+| `Large.zip` | Yes | Yes |
+| Generated `Large.tar` | Yes | N/A |
+| `Large.gz` | Yes | Yes |
+| `Large.rar` | Yes | Yes |
+| `Large.7z` | Yes | No |
+| `Large.tar.gz` | No | Yes |
+
+The compressible payload keeps the compressed fixtures small while requiring a full
+64 MiB decompression to validate each API. The Archive API test expands `Large.tar.gz`
+to a scratch `Large.tar` before exercising raw TAR support, so the 64 MiB TAR file is
+not committed.
+
+## Regenerating
+
+The fixtures were created with `zip`, `tar`, `gzip`, RAR 7.22, and 7-Zip. Run these
+commands from a temporary directory after replacing `<repo>` with the repository root:
+
+```sh
+yes "SharpCompress large fixture" | head -c 67108864 > large.bin
+printf '\377%.0s' {1..2048} > prefix.bin
+dd if=prefix.bin of=large.bin bs=2048 count=1 conv=notrunc
+touch -t 202001010000 large.bin
+mkdir -p <repo>/tests/TestArchives/Archives/Large
+zip -X -9 -j <repo>/tests/TestArchives/Archives/Large/Large.zip large.bin
+COPYFILE_DISABLE=1 tar -cf large.tar large.bin
+gzip -9 -c large.bin > <repo>/tests/TestArchives/Archives/Large/Large.gz
+gzip -n -9 -c large.tar > <repo>/tests/TestArchives/Archives/Large/Large.tar.gz
+rar a -ma5 -m5 -ep <repo>/tests/TestArchives/Archives/Large/Large.rar large.bin
+7z a -t7z -mx=9 <repo>/tests/TestArchives/Archives/Large/Large.7z large.bin
+```