deflate 0 optimizations

src/SharpCompress/Compressors/Deflate/DeflateManager.cs

@@ -304,6 +304,7 @@ internal sealed partial class DeflateManager
     // An index in this array is thus a window index modulo 32K.
 
     private short[] head; // Heads of the hash chains or NIL.
+    private bool hashChainsPopulated;
 
     private int ins_h; // hash index of string to be inserted
     private int hash_size; // number of elements in hash table
@@ -419,6 +420,7 @@ internal sealed partial class DeflateManager
 
         // clear the hash - workitem 9063
         Array.Clear(head, 0, hash_size);
+        hashChainsPopulated = false;
 
         //for (int i = 0; i < hash_size; i++) head[i] = 0;
 
@@ -1206,30 +1208,29 @@ internal sealed partial class DeflateManager
                 strstart -= w_size; // we now have strstart >= MAX_DIST
                 blockStart -= w_size;
 
-                // Slide the hash table (could be avoided with 32 bit values
-                // at the expense of memory usage). We slide even when level == 0
-                // to keep the hash table consistent if we switch back to level > 0
-                // later. (Using level 0 permanently is not an optimal usage of
-                // zlib, so we don't care about this pathological case.)
-
-                n = hash_size;
-                p = n;
-                do
+                // Stored blocks never add hash entries. Until a dictionary or a
+                // matching deflater has used the tables, head remains all NIL and
+                // prev is unreachable, so sliding either table is wasted work.
+                // Once populated, keep sliding even at level 0: SetParams may
+                // switch back to matching and must see the same history.
+                if (hashChainsPopulated)
                 {
-                    m = (head[--p] & 0xffff);
-                    head[p] = (short)((m >= w_size) ? (m - w_size) : 0);
-                } while (--n != 0);
+                    n = hash_size;
+                    p = n;
+                    do
+                    {
+                        m = (head[--p] & 0xffff);
+                        head[p] = (short)((m >= w_size) ? (m - w_size) : 0);
+                    } while (--n != 0);
 
-                n = w_size;
-                p = n;
-                do
-                {
-                    m = (prev[--p] & 0xffff);
-                    prev[p] = (short)((m >= w_size) ? (m - w_size) : 0);
-
-                    // If n is not on any hash chain, prev[n] is garbage but
-                    // its value will never be used.
-                } while (--n != 0);
+                    n = w_size;
+                    p = n;
+                    do
+                    {
+                        m = (prev[--p] & 0xffff);
+                        prev[p] = (short)((m >= w_size) ? (m - w_size) : 0);
+                    } while (--n != 0);
+                }
                 more += w_size;
             }
 
@@ -1271,6 +1272,7 @@ internal sealed partial class DeflateManager
     // matches. It is used only for the fast compression options.
     internal BlockState DeflateFast(FlushType flush)
     {
+        hashChainsPopulated = true;
         //    short hash_head = 0; // head of the hash chain
         var hash_head = 0; // head of the hash chain
         bool bflush; // set if current block must be flushed
@@ -1401,6 +1403,7 @@ internal sealed partial class DeflateManager
     // no better match at the next window position.
     internal BlockState DeflateSlow(FlushType flush)
     {
+        hashChainsPopulated = true;
         //    short hash_head = 0;    // head of hash chain
         var hash_head = 0; // head of hash chain
         bool bflush; // set if current block must be flushed
@@ -1887,6 +1890,7 @@ internal sealed partial class DeflateManager
         // s->lookahead stays null, so s->ins_h will be recomputed at the next
         // call of fill_window.
 
+        hashChainsPopulated = true;
         ins_h = window[0] & 0xff;
         ins_h = (((ins_h) << hash_shift) ^ (window[1] & 0xff)) & hash_mask;
 

tests/SharpCompress.Test/Streams/DeflateStoredWindowTests.cs

@@ -0,0 +1,215 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using SharpCompress.Compressors;
+using SharpCompress.Compressors.Deflate;
+using SharpCompress.Test.Mocks;
+using Xunit;
+
+namespace SharpCompress.Test.Streams;
+
+public class DeflateStoredWindowTests
+{
+    [Theory]
+    [InlineData(false, 1)]
+    [InlineData(true, 1)]
+    [InlineData(false, 32767)]
+    [InlineData(true, 32767)]
+    [InlineData(false, 65536)]
+    [InlineData(true, 65536)]
+    public async Task Stored_Blocks_RoundTrip_Across_Window_Slides(bool useAsync, int chunkSize)
+    {
+        var data = CreateData(256 * 1024 + 17);
+        using var output = new MemoryStream();
+        if (useAsync)
+        {
+            await using var compressor = new DeflateStream(
+                new AsyncOnlyStream(output, disposeStream: false),
+                CompressionMode.Compress,
+                CompressionLevel.None
+            );
+            for (var offset = 0; offset < data.Length; offset += chunkSize)
+            {
+                await compressor.WriteAsync(
+                    data,
+                    offset,
+                    Math.Min(chunkSize, data.Length - offset),
+                    CancellationToken.None
+                );
+            }
+        }
+        else
+        {
+            using var compressor = new DeflateStream(
+                new ForwardOnlyStream(output),
+                CompressionMode.Compress,
+                CompressionLevel.None
+            );
+            for (var offset = 0; offset < data.Length; offset += chunkSize)
+            {
+                compressor.Write(data, offset, Math.Min(chunkSize, data.Length - offset));
+            }
+        }
+        AssertRawRoundTrip(output.ToArray(), data);
+    }
+
+    [Theory]
+    [InlineData(false, FlushType.None)]
+    [InlineData(false, FlushType.Partial)]
+    [InlineData(false, FlushType.Sync)]
+    [InlineData(false, FlushType.Full)]
+    [InlineData(true, FlushType.None)]
+    [InlineData(true, FlushType.Partial)]
+    [InlineData(true, FlushType.Sync)]
+    [InlineData(true, FlushType.Full)]
+    public void Changing_Levels_Preserves_History_Across_Stored_Windows(
+        bool dictionary,
+        FlushType flush
+    )
+    {
+        // Each segment crosses several windows. Returning to level 0 after matching
+        // has populated the tables must still slide the existing match history.
+        var segment = CreateData(128 * 1024 + 13);
+        var levels = new[]
+        {
+            CompressionLevel.None,
+            CompressionLevel.BestSpeed,
+            CompressionLevel.None,
+            CompressionLevel.Default,
+            CompressionLevel.None,
+            CompressionLevel.BestCompression,
+        };
+        var expected = levels.SelectMany(_ => segment).ToArray();
+        var compressed = new byte[expected.Length * 2 + 65536];
+        var codec = new ZlibCodec();
+        var preset = segment.Take(32000).ToArray();
+        Assert.Equal(0, codec.InitializeDeflate(levels[0], wantRfc1950Header: dictionary));
+        try
+        {
+            if (dictionary)
+            {
+                Assert.Equal(0, codec.SetDictionary(preset));
+            }
+            codec.OutputBuffer = compressed;
+            codec.AvailableBytesOut = compressed.Length;
+            foreach (var level in levels)
+            {
+                Assert.Equal(0, codec.SetDeflateParams(level, CompressionStrategy.Default));
+                codec.InputBuffer = segment;
+                codec.NextIn = 0;
+                codec.AvailableBytesIn = segment.Length;
+                do
+                {
+                    Assert.Equal(0, codec.Deflate(flush));
+                    Assert.True(codec.AvailableBytesOut > 0);
+                } while (codec.AvailableBytesIn > 0);
+            }
+            Assert.Equal(1, codec.Deflate(FlushType.Finish));
+            var bytes = compressed.Take(codec.NextOut).ToArray();
+            if (dictionary)
+            {
+                AssertDictionaryRoundTrip(bytes, preset, expected);
+            }
+            else
+            {
+                AssertRawRoundTrip(bytes, expected);
+            }
+        }
+        finally
+        {
+            codec.EndDeflate();
+        }
+    }
+
+    [Theory]
+    [InlineData(9)]
+    [InlineData(15)]
+    public void Reset_Then_Stored_Compression_Discards_Previous_Hash_History(int windowBits)
+    {
+        var data = CreateData(256 * 1024 + 17);
+        var codec = new ZlibCodec();
+        Assert.Equal(0, codec.InitializeDeflate(CompressionLevel.Default, windowBits, false));
+        try
+        {
+            foreach (var level in new[] { CompressionLevel.Default, CompressionLevel.None })
+            {
+                Assert.Equal(0, codec.SetDeflateParams(level, CompressionStrategy.Default));
+                codec.InputBuffer = data;
+                codec.NextIn = 0;
+                codec.AvailableBytesIn = data.Length;
+                codec.OutputBuffer = new byte[data.Length * 2];
+                codec.NextOut = 0;
+                codec.AvailableBytesOut = codec.OutputBuffer.Length;
+                Assert.Equal(1, codec.Deflate(FlushType.Finish));
+                AssertRawRoundTrip(codec.OutputBuffer.Take(codec.NextOut).ToArray(), data);
+                codec.ResetDeflate();
+            }
+        }
+        finally
+        {
+            codec.EndDeflate();
+        }
+    }
+
+    private static void AssertDictionaryRoundTrip(
+        byte[] compressed,
+        byte[] dictionary,
+        byte[] expected
+    )
+    {
+        var codec = new ZlibCodec();
+        Assert.Equal(0, codec.InitializeInflate());
+        try
+        {
+            codec.InputBuffer = compressed;
+            codec.AvailableBytesIn = compressed.Length;
+            codec.OutputBuffer = new byte[expected.Length + 1];
+            codec.AvailableBytesOut = codec.OutputBuffer.Length;
+            Assert.Equal(2, codec.Inflate(FlushType.None));
+            Assert.Equal(0, codec.SetDictionary(dictionary));
+            Assert.Equal(1, codec.Inflate(FlushType.Finish));
+            Assert.Equal(expected.Length, codec.TotalBytesOut);
+            Assert.Equal(expected, codec.OutputBuffer.Take(expected.Length).ToArray());
+        }
+        finally
+        {
+            codec.EndInflate();
+        }
+    }
+
+    private static void AssertRawRoundTrip(byte[] compressed, byte[] expected)
+    {
+        using (var source = new MemoryStream(compressed))
+        using (var decompressor = new DeflateStream(source, CompressionMode.Decompress))
+        using (var extracted = new MemoryStream())
+        {
+            decompressor.CopyTo(extracted);
+            Assert.Equal(expected, extracted.ToArray());
+        }
+#if !LEGACY_DOTNET
+        using var nativeSource = new MemoryStream(compressed);
+        using var native = new System.IO.Compression.DeflateStream(
+            nativeSource,
+            System.IO.Compression.CompressionMode.Decompress
+        );
+        using var nativeOutput = new MemoryStream();
+        native.CopyTo(nativeOutput);
+        Assert.Equal(expected, nativeOutput.ToArray());
+#endif
+    }
+
+    private static byte[] CreateData(int length)
+    {
+        var data = new byte[length];
+        var random = new Random(1234);
+        random.NextBytes(data);
+        // Mix random bytes with repeatable matches, including near window boundaries.
+        for (var offset = 4096; offset + 4096 <= length; offset += 8192)
+        {
+            Array.Copy(data, 0, data, offset, 4096);
+        }
+        return data;
+    }
+}