From 8174af616f5773b5650938affccceade6fb29ed9 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 22 Sep 2023 11:53:27 -0400 Subject: [PATCH] Handle model issues, combine logic --- ...Decompressor.cs => DeflateDecompressor.cs} | 218 ++++++++++-------- MSZIP/HuffmanDecoder.cs | 76 +++++- 2 files changed, 196 insertions(+), 98 deletions(-) rename MSZIP/{Decompressor.cs => DeflateDecompressor.cs} (69%) diff --git a/MSZIP/Decompressor.cs b/MSZIP/DeflateDecompressor.cs similarity index 69% rename from MSZIP/Decompressor.cs rename to MSZIP/DeflateDecompressor.cs index d3f5e91..8d62244 100644 --- a/MSZIP/Decompressor.cs +++ b/MSZIP/DeflateDecompressor.cs @@ -7,9 +7,8 @@ using static SabreTools.Compression.MSZIP.Constants; namespace SabreTools.Compression.MSZIP { - // TODO: Combine parts of the fixed and dynamic implementation /// - public class Decompressor + public class DeflateDecompressor { /// /// Internal bitstream to use for decompression @@ -20,7 +19,7 @@ namespace SabreTools.Compression.MSZIP /// Create a new Decompressor from a Stream /// /// Stream to decompress - public Decompressor(Stream input) + public DeflateDecompressor(Stream input) { // If we have an invalid stream if (input == null || !input.CanRead || !input.CanSeek) @@ -116,6 +115,52 @@ namespace SabreTools.Compression.MSZIP return header; } + /// + /// Read a FixedHuffmanCompressedBlockHeader from the input stream + /// + private FixedHuffmanCompressedBlockHeader ReadFixedHuffmanCompressedBlockHeader() + { + return new FixedHuffmanCompressedBlockHeader(); + } + + /// + /// Read a DynamicHuffmanCompressedBlockHeader from the input stream + /// + private (DynamicHuffmanCompressedBlockHeader, uint, uint) ReadDynamicHuffmanCompressedBlockHeader() + { + var header = new DynamicHuffmanCompressedBlockHeader(); + + // Setup the counts first + uint numLiteral = 257 + _bitStream.ReadBitsLSB(5) ?? 0; + uint numDistance = 1 + _bitStream.ReadBitsLSB(5) ?? 0; + uint numLength = 4 + _bitStream.ReadBitsLSB(4) ?? 0; + + // Convert the alphabet based on lengths + uint[] lengthLengths = new uint[19]; + for (int i = 0; i < numLength; i++) + { + lengthLengths[FixedAlphabet[i]] = (byte)_bitStream.ReadBitsLSB(3); + } + for (int i = (int)numLength; i < 19; i++) + { + lengthLengths[FixedAlphabet[i]] = 0; + } + + // Make the lengths tree + HuffmanDecoder lengthTree = new HuffmanDecoder(lengthLengths, 19); + + // Setup the literal and distance lengths + header.LiteralLengths = new int[288]; + header.DistanceCodes = new int[32]; + + // Read the literal and distance codes + int repeatCode = 1; + uint leftover = ReadHuffmanLengths(lengthTree, header.LiteralLengths, numLiteral, 0, ref repeatCode); + _ = ReadHuffmanLengths(lengthTree, header.DistanceCodes, numDistance, leftover, ref repeatCode); + + return (header, numLiteral, numDistance); + } + #endregion #region Data @@ -144,65 +189,15 @@ namespace SabreTools.Compression.MSZIP { var bytes = new List(); - while (true) - { - // Read the next symbol from the bitstream - uint? sym = _bitStream.ReadBitsMSB(7); - if (sym == null) - return null; + // Get the fixed huffman header + var header = ReadFixedHuffmanCompressedBlockHeader(); - // If we have a symbol number <= 23, adjust it - if (sym <= 23) - { - sym += 256; - } + // Make the literal and distance trees + HuffmanDecoder literalTree = new HuffmanDecoder(header.LiteralLengths, 288); + HuffmanDecoder distanceTree = new HuffmanDecoder(header.DistanceCodes, 30); - // Read the next bit otherwise - else - { - sym = (sym << 1) + _bitStream.ReadBit(); - if (sym <= 191) - sym -= 48; - else if (sym <= 199) - sym += 88; - else - sym = (sym << 1) + _bitStream.ReadBit() - 256; - } - - // If we have an immediate symbol - if (sym < 256) - { - bytes.Add((byte)sym); - } - - // If we have the ending symbol - else if (sym == 256) - { - break; - } - - // If we have a length/distance pair - else - { - sym -= 257; - uint? length = MatchLengths[(int)sym] + _bitStream.ReadBitsLSB(MatchExtraBits[(int)sym]); - if (length == null) - return null; - - uint? distanceCode = _bitStream.ReadBitsMSB(5); - if (distanceCode == null) - return null; - - uint? distance = DistanceLengths[(int)distanceCode] + _bitStream.ReadBitsLSB(DistanceExtraBits[(int)distanceCode]); - if (distance == null) - return null; - - byte[] arr = bytes.Skip(bytes.Count - (int)distance).Take((int)length).ToArray(); - bytes.AddRange(arr); - } - } - - return bytes.ToArray(); + // Now loop and decode + return ReadHuffmanBlock(literalTree, distanceTree); } /// @@ -210,38 +205,22 @@ namespace SabreTools.Compression.MSZIP /// private byte[] ReadDynamicHuffman() { - // Setup the counts first - uint numLiteral = 257 + _bitStream.ReadBitsLSB(5) ?? 0; - uint numDistance = 1 + _bitStream.ReadBitsLSB(5) ?? 0; - uint numLength = 4 + _bitStream.ReadBitsLSB(4) ?? 0; - - // Convert the alphabet based on lengths - byte[] lengthLengths = new byte[19]; - for (int i = 0; i < numLength; i++) - { - lengthLengths[FixedAlphabet[i]] = (byte)_bitStream.ReadBitsLSB(3); - } - for (int i = (int)numLength; i < 19; i++) - { - lengthLengths[FixedAlphabet[i]] = 0; - } - - // Make the lengths tree - HuffmanDecoder lengthTree = new HuffmanDecoder(lengthLengths, 19); - - // Setup the literal and distance lengths - byte[] literalLengths = new byte[288]; - byte[] distanceLengths = new byte[32]; - - // Read the literal and distance codes - int repeatCode = 1; - uint leftover = ReadHuffmanLengths(lengthTree, literalLengths, numLiteral, 0, ref repeatCode); - _ = ReadHuffmanLengths(lengthTree, distanceLengths, numDistance, leftover, ref repeatCode); + // Get the dynamic huffman header + (var header, uint numLiteral, uint numDistance) = ReadDynamicHuffmanCompressedBlockHeader(); // Make the literal and distance trees - HuffmanDecoder literalTree = new HuffmanDecoder(literalLengths, numLiteral); - HuffmanDecoder distanceTree = new HuffmanDecoder(distanceLengths, numDistance); + HuffmanDecoder literalTree = new HuffmanDecoder(header.LiteralLengths, numLiteral); + HuffmanDecoder distanceTree = new HuffmanDecoder(header.DistanceCodes, numDistance); + // Now loop and decode + return ReadHuffmanBlock(literalTree, distanceTree); + } + + /// + /// Read an RFC1951 block with Huffman compression + /// + private byte[] ReadHuffmanBlock(HuffmanDecoder literalTree, HuffmanDecoder distanceTree) + { // Now loop and decode var bytes = new List(); while (true) @@ -287,7 +266,66 @@ namespace SabreTools.Compression.MSZIP /// /// Read the huffman lengths /// - private uint ReadHuffmanLengths(HuffmanDecoder lengthTree, byte[] lengths, uint numCodes, uint repeat, ref int repeatCode) + private uint ReadHuffmanLengths(HuffmanDecoder lengthTree, int[] lengths, uint numCodes, uint repeat, ref int repeatCode) + { + int i = 0; + + // First fill in any repeat codes + while (repeat > 0) + { + lengths[i++] = (byte)repeatCode; + repeat--; + } + + // Then process the rest of the table + while (i < numCodes) + { + // Get the next length encoding from the stream + int lengthEncoding = lengthTree.Decode(_bitStream); + + // Values less than 16 are encoded directly + if (lengthEncoding < 16) + { + lengths[i++] = (byte)lengthEncoding; + repeatCode = lengthEncoding; + } + + // Otherwise, the repeat count is based on the next values + else + { + // Determine the repeat count and code from the encoding + if (lengthEncoding == 16) + { + repeat = 3 + _bitStream.ReadBitsLSB(2) ?? 0; + } + else if (lengthEncoding == 17) + { + repeat = 3 + _bitStream.ReadBitsLSB(3) ?? 0; + repeatCode = 0; + } + else if (lengthEncoding == 18) + { + repeat = 11 + _bitStream.ReadBitsLSB(7) ?? 0; + repeatCode = 0; + } + + // Read in the expected lengths + while (i < numCodes && repeat > 0) + { + lengths[i++] = (byte)repeatCode; + repeat--; + } + } + } + + // Return any repeat value we have left over + return repeat; + } + + /// + /// Read the huffman lengths + /// + private uint ReadHuffmanLengths(HuffmanDecoder lengthTree, uint[] lengths, uint numCodes, uint repeat, ref int repeatCode) { int i = 0; diff --git a/MSZIP/HuffmanDecoder.cs b/MSZIP/HuffmanDecoder.cs index cec0741..efd0eac 100644 --- a/MSZIP/HuffmanDecoder.cs +++ b/MSZIP/HuffmanDecoder.cs @@ -15,19 +15,19 @@ namespace SabreTools.Compression.MSZIP /// /// Array representing the number of bits for each value /// Number of Huffman codes encoded - public HuffmanDecoder(byte[] lengths, uint numCodes) + public HuffmanDecoder(int[] lengths, uint numCodes) { // Set the root to null for now _root = null; // Determine the value for max_bits - int max_bits = lengths.Max(); + uint max_bits = (uint)lengths.Max(); // Count the number of codes for each code length int[] bl_count = new int[max_bits + 1]; for (int i = 0; i < numCodes; i++) { - int length = lengths[i]; + uint length = (uint)lengths[i]; bl_count[length]++; } @@ -48,7 +48,7 @@ namespace SabreTools.Compression.MSZIP int[] tree = new int[numCodes]; for (int i = 0; i < numCodes; i++) { - byte len = lengths[i]; + uint len = (uint)lengths[i]; if (len == 0) continue; @@ -61,7 +61,67 @@ namespace SabreTools.Compression.MSZIP for (int i = 0; i < numCodes; i++) { // If we have a 0-length code - byte len = lengths[i]; + uint len = (uint)lengths[i]; + if (len == 0) + continue; + + // Insert the value starting at the root + _root = Insert(_root, i, len, tree[i]); + } + } + + /// + /// Create a Huffman tree to decode with + /// + /// Array representing the number of bits for each value + /// Number of Huffman codes encoded + public HuffmanDecoder(uint[] lengths, uint numCodes) + { + // Set the root to null for now + _root = null; + + // Determine the value for max_bits + uint max_bits = lengths.Max(); + + // Count the number of codes for each code length + int[] bl_count = new int[max_bits + 1]; + for (int i = 0; i < numCodes; i++) + { + uint length = lengths[i]; + bl_count[length]++; + } + + // Find the numerical value of the smalles code for each code length + int[] next_code = new int[max_bits + 1]; + int code = 0; + bl_count[0] = 0; + for (int bits = 1; bits <= max_bits; bits++) + { + code = (code + bl_count[bits - 1]) << 1; + next_code[bits] = code; + } + + // Assign numerical values to all codes, using consecutive + // values for all codes of the same length with the base + // values determined at step 2. Codes that are never used + // (which have a bit length of zero) must not be assigned a value. + int[] tree = new int[numCodes]; + for (int i = 0; i < numCodes; i++) + { + uint len = lengths[i]; + if (len == 0) + continue; + + // Set the value in the tree + tree[i] = next_code[len]; + next_code[len]++; + } + + // Now insert the values into the structure + for (int i = 0; i < numCodes; i++) + { + // If we have a 0-length code + uint len = lengths[i]; if (len == 0) continue; @@ -106,9 +166,9 @@ namespace SabreTools.Compression.MSZIP /// Encoding of the value to traverse /// New instance of the node with value appended #if NET48 - private static HuffmanNode Insert(HuffmanNode node, int value, int length, int code) + private static HuffmanNode Insert(HuffmanNode node, int value, uint length, int code) #else - private static HuffmanNode Insert(HuffmanNode? node, int value, int length, int code) + private static HuffmanNode Insert(HuffmanNode? node, int value, uint length, int code) #endif { // If no node is provided, create a new one @@ -123,7 +183,7 @@ namespace SabreTools.Compression.MSZIP } // Otherwise, get the next bit from the code - byte nextBit = (byte)(code >> (length - 1) & 1); + byte nextBit = (byte)(code >> (int)(length - 1) & 1); // Left == 0, Right == 1 if (nextBit == 0)