diff --git a/BitStream.cs b/BitStream.cs index fb1ad71..886c3b2 100644 --- a/BitStream.cs +++ b/BitStream.cs @@ -126,23 +126,18 @@ namespace SabreTools.Compression /// Read a byte, if possible /// /// The next byte, null on error or end of stream + /// Assumes the stream is byte-aligned public byte? ReadByte() { - // If we don't have a value cached - if (_bitBuffer == null) + try { - try - { - return _source.ReadByteValue(); - } - catch - { - return null; - } + Discard(); + return _source.ReadByteValue(); + } + catch + { + return null; } - - // Otherwise, assemble the value from the next bits - throw new NotImplementedException(); } /// @@ -154,6 +149,7 @@ namespace SabreTools.Compression { try { + Discard(); return _source.ReadUInt16(); } catch @@ -171,6 +167,7 @@ namespace SabreTools.Compression { try { + Discard(); return _source.ReadUInt32(); } catch @@ -188,6 +185,7 @@ namespace SabreTools.Compression { try { + Discard(); return _source.ReadUInt64(); } catch @@ -206,6 +204,7 @@ namespace SabreTools.Compression { try { + Discard(); return _source.ReadBytes(bytes); } catch diff --git a/MSZIP/Constants.cs b/MSZIP/Constants.cs new file mode 100644 index 0000000..69df732 --- /dev/null +++ b/MSZIP/Constants.cs @@ -0,0 +1,53 @@ +namespace SabreTools.Compression.MSZIP +{ + public static class Constants + { + /// + /// Alphabet for fixed Huffman encoding + /// + public static readonly byte[] FixedAlphabet = new byte[19] + { + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, + }; + + /// + /// Extra bits for length codes 257-285 + /// + public static readonly byte[] MatchExtraBits = new byte[29] + { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 0, + }; + + /// + /// Initial lengths for length codes 257-285 + /// + public static readonly ushort[] MatchLengths = new ushort[29] + { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, + 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, + 67, 83, 99, 115, 131, 163, 195, 227, 258, + }; + + /// + /// Extra bits for distance codes 0-29 + /// + public static readonly byte[] DistanceExtraBits = new byte[30] + { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + }; + + /// + /// Initial lengths for distance codes 0-29 + /// + public static readonly ushort[] DistanceLengths = new ushort[30] + { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, + 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, + 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 + }; + } +} \ No newline at end of file diff --git a/MSZIP/Decompressor.cs b/MSZIP/Decompressor.cs index 644bc42..d3f5e91 100644 --- a/MSZIP/Decompressor.cs +++ b/MSZIP/Decompressor.cs @@ -1,29 +1,43 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using SabreTools.Models.Compression.MSZIP; +using static SabreTools.Compression.MSZIP.Constants; namespace SabreTools.Compression.MSZIP { + // TODO: Combine parts of the fixed and dynamic implementation + /// public class Decompressor { /// - /// Decompress a stream into a byte array + /// Internal bitstream to use for decompression + /// + private BitStream _bitStream; + + /// + /// Create a new Decompressor from a Stream /// /// Stream to decompress - /// Byte array containing the decompressed data on success, null on error - /// - public static byte[] Decompress(Stream input) + public Decompressor(Stream input) { // If we have an invalid stream if (input == null || !input.CanRead || !input.CanSeek) - return null; + throw new ArgumentException(nameof(input)); // Wrap the stream in a BitStream - var bitStream = new BitStream(input); + _bitStream = new BitStream(input); + } + /// + /// Decompress a stream into a byte array + /// + /// Byte array containing the decompressed data on success, null on error + public byte[] Process() + { // Try to read the header - var blockHeader = ReadBlockHeader(bitStream); + var blockHeader = ReadBlockHeader(); if (blockHeader.Signature != 0x4B43) return null; @@ -32,79 +46,61 @@ namespace SabreTools.Compression.MSZIP while (true) { // Try to read the deflate block header - var deflateBlockHeader = ReadDeflateBlockHeader(bitStream); - if (deflateBlockHeader.BTYPE == CompressionType.Reserved) - break; - - // If stored with no compression - if (deflateBlockHeader.BTYPE == CompressionType.NoCompression) + var deflateBlockHeader = ReadDeflateBlockHeader(); + switch (deflateBlockHeader.BTYPE) { - // Skip any remaining bits in current partially processed byte - bitStream.Discard(); - - // Read LEN and NLEN - var nonCompressedBlockHeader = ReadNonCompressedBlockHeader(bitStream); - if (nonCompressedBlockHeader.LEN == 0 && nonCompressedBlockHeader.NLEN == 0) + // If stored with no compression + case CompressionType.NoCompression: + byte[] bytes00 = ReadNoCompression(); + bytes.AddRange(bytes00); break; - // Copy LEN bytes of data to output - byte[] uncompressed = bitStream.ReadBytes(nonCompressedBlockHeader.LEN); - bytes.AddRange(uncompressed); - } + // If compressed with fixed Huffman codes + case CompressionType.FixedHuffman: + byte[] bytes01 = ReadFixedHuffman(); + bytes.AddRange(bytes01); + break; - // Otherwise - else - { // If compressed with dynamic Huffman codes - if (deflateBlockHeader.BTYPE == CompressionType.DynamicHuffman) - { - // Read representation of code trees - } + case CompressionType.DynamicHuffman: + byte[] bytes10 = ReadDynamicHuffman(); + bytes.AddRange(bytes10); + break; - // Loop (until end of block code recognized) - while (true) - { - /* - decode literal/length value from input stream - if value < 256 - copy value (literal byte) to output stream - otherwise - if value = end of block (256) - break from loop - otherwise (value = 257..285) - decode distance from input stream - - move backwards distance bytes in the output - stream, and copy length bytes from this - position to the output stream. - */ - } + // Reserved is not allowed and is treated as an error + case CompressionType.Reserved: + default: + return null; } + + // If we're at the final block, exit out of the loop + if (deflateBlockHeader.BFINAL) + break; } - throw new NotImplementedException(); + return bytes.ToArray(); } + #region Headers + /// /// Read a BlockHeader from the input stream /// - private static BlockHeader ReadBlockHeader(BitStream input) + private BlockHeader ReadBlockHeader() { var header = new BlockHeader(); - header.Signature = input.ReadUInt16() ?? 0; + header.Signature = _bitStream.ReadUInt16() ?? 0; return header; } /// /// Read a DeflateBlockHeader from the input stream /// - private static DeflateBlockHeader ReadDeflateBlockHeader(BitStream input) + private DeflateBlockHeader ReadDeflateBlockHeader() { var header = new DeflateBlockHeader(); - header.BFINAL = input.ReadBit() != 0x01; - byte btype = input.ReadBit() ?? 0x01; - btype <<= 1; - btype |= input.ReadBit() ?? 0x01; + header.BFINAL = _bitStream.ReadBit() != 0x01; + uint? btype = _bitStream.ReadBitsLSB(2) ?? 0b11; header.BTYPE = (CompressionType)btype; return header; } @@ -112,12 +108,241 @@ namespace SabreTools.Compression.MSZIP /// /// Read a NonCompressedBlockHeader from the input stream /// - private static NonCompressedBlockHeader ReadNonCompressedBlockHeader(BitStream input) + private NonCompressedBlockHeader ReadNonCompressedBlockHeader() { var header = new NonCompressedBlockHeader(); - header.LEN = input.ReadUInt16() ?? 0; - header.NLEN = input.ReadUInt16() ?? 0; + header.LEN = _bitStream.ReadUInt16() ?? 0; + header.NLEN = _bitStream.ReadUInt16() ?? 0; return header; } + + #endregion + + #region Data + + /// + /// Read an RFC1951 block with no compression + /// + private byte[] ReadNoCompression() + { + // Skip any remaining bits in current partially processed byte + _bitStream.Discard(); + + // Read LEN and NLEN + var nonCompressedBlockHeader = ReadNonCompressedBlockHeader(); + if (nonCompressedBlockHeader.LEN == 0 && nonCompressedBlockHeader.NLEN == 0) + return null; + + // Copy LEN bytes of data to output + return _bitStream.ReadBytes(nonCompressedBlockHeader.LEN); + } + + /// + /// Read an RFC1951 block with fixed Huffman compression + /// + private byte[] ReadFixedHuffman() + { + var bytes = new List(); + + while (true) + { + // Read the next symbol from the bitstream + uint? sym = _bitStream.ReadBitsMSB(7); + if (sym == null) + return null; + + // If we have a symbol number <= 23, adjust it + if (sym <= 23) + { + sym += 256; + } + + // Read the next bit otherwise + else + { + sym = (sym << 1) + _bitStream.ReadBit(); + if (sym <= 191) + sym -= 48; + else if (sym <= 199) + sym += 88; + else + sym = (sym << 1) + _bitStream.ReadBit() - 256; + } + + // If we have an immediate symbol + if (sym < 256) + { + bytes.Add((byte)sym); + } + + // If we have the ending symbol + else if (sym == 256) + { + break; + } + + // If we have a length/distance pair + else + { + sym -= 257; + uint? length = MatchLengths[(int)sym] + _bitStream.ReadBitsLSB(MatchExtraBits[(int)sym]); + if (length == null) + return null; + + uint? distanceCode = _bitStream.ReadBitsMSB(5); + if (distanceCode == null) + return null; + + uint? distance = DistanceLengths[(int)distanceCode] + _bitStream.ReadBitsLSB(DistanceExtraBits[(int)distanceCode]); + if (distance == null) + return null; + + byte[] arr = bytes.Skip(bytes.Count - (int)distance).Take((int)length).ToArray(); + bytes.AddRange(arr); + } + } + + return bytes.ToArray(); + } + + /// + /// Read an RFC1951 block with dynamic Huffman compression + /// + private byte[] ReadDynamicHuffman() + { + // Setup the counts first + uint numLiteral = 257 + _bitStream.ReadBitsLSB(5) ?? 0; + uint numDistance = 1 + _bitStream.ReadBitsLSB(5) ?? 0; + uint numLength = 4 + _bitStream.ReadBitsLSB(4) ?? 0; + + // Convert the alphabet based on lengths + byte[] lengthLengths = new byte[19]; + for (int i = 0; i < numLength; i++) + { + lengthLengths[FixedAlphabet[i]] = (byte)_bitStream.ReadBitsLSB(3); + } + for (int i = (int)numLength; i < 19; i++) + { + lengthLengths[FixedAlphabet[i]] = 0; + } + + // Make the lengths tree + HuffmanDecoder lengthTree = new HuffmanDecoder(lengthLengths, 19); + + // Setup the literal and distance lengths + byte[] literalLengths = new byte[288]; + byte[] distanceLengths = new byte[32]; + + // Read the literal and distance codes + int repeatCode = 1; + uint leftover = ReadHuffmanLengths(lengthTree, literalLengths, numLiteral, 0, ref repeatCode); + _ = ReadHuffmanLengths(lengthTree, distanceLengths, numDistance, leftover, ref repeatCode); + + // Make the literal and distance trees + HuffmanDecoder literalTree = new HuffmanDecoder(literalLengths, numLiteral); + HuffmanDecoder distanceTree = new HuffmanDecoder(distanceLengths, numDistance); + + // Now loop and decode + var bytes = new List(); + while (true) + { + // Decode the next literal value + int sym = literalTree.Decode(_bitStream); + + // If we have an immediate symbol + if (sym < 256) + { + bytes.Add((byte)sym); + } + + // If we have the ending symbol + else if (sym == 256) + { + break; + } + + // If we have a length/distance pair + else + { + sym -= 257; + uint? length = MatchLengths[sym] + _bitStream.ReadBitsLSB(MatchExtraBits[sym]); + if (length == null) + return null; + + int distanceCode = distanceTree.Decode(_bitStream); + + uint? distance = DistanceLengths[distanceCode] + _bitStream.ReadBitsLSB(DistanceExtraBits[distanceCode]); + if (distance == null) + return null; + + byte[] arr = bytes.Skip(bytes.Count - (int)distance).Take((int)length).ToArray(); + bytes.AddRange(arr); + } + } + + // Return the decoded array + return bytes.ToArray(); + } + + /// + /// Read the huffman lengths + /// + private uint ReadHuffmanLengths(HuffmanDecoder lengthTree, byte[] lengths, uint numCodes, uint repeat, ref int repeatCode) + { + int i = 0; + + // First fill in any repeat codes + while (repeat > 0) + { + lengths[i++] = (byte)repeatCode; + repeat--; + } + + // Then process the rest of the table + while (i < numCodes) + { + // Get the next length encoding from the stream + int lengthEncoding = lengthTree.Decode(_bitStream); + + // Values less than 16 are encoded directly + if (lengthEncoding < 16) + { + lengths[i++] = (byte)lengthEncoding; + repeatCode = lengthEncoding; + } + + // Otherwise, the repeat count is based on the next values + else + { + // Determine the repeat count and code from the encoding + if (lengthEncoding == 16) + { + repeat = 3 + _bitStream.ReadBitsLSB(2) ?? 0; + } + else if (lengthEncoding == 17) + { + repeat = 3 + _bitStream.ReadBitsLSB(3) ?? 0; + repeatCode = 0; + } + else if (lengthEncoding == 18) + { + repeat = 11 + _bitStream.ReadBitsLSB(7) ?? 0; + repeatCode = 0; + } + + // Read in the expected lengths + while (i < numCodes && repeat > 0) + { + lengths[i++] = (byte)repeatCode; + repeat--; + } + } + } + + // Return any repeat value we have left over + return repeat; + } + + #endregion } } \ No newline at end of file diff --git a/MSZIP/HuffmanDecoder.cs b/MSZIP/HuffmanDecoder.cs index 186b997..cec0741 100644 --- a/MSZIP/HuffmanDecoder.cs +++ b/MSZIP/HuffmanDecoder.cs @@ -14,8 +14,8 @@ namespace SabreTools.Compression.MSZIP /// Create a Huffman tree to decode with /// /// Array representing the number of bits for each value - /// Number of Huffman codes encoded - public HuffmanDecoder(byte[] lengths, uint num_codes) + /// Number of Huffman codes encoded + public HuffmanDecoder(byte[] lengths, uint numCodes) { // Set the root to null for now _root = null; @@ -25,7 +25,7 @@ namespace SabreTools.Compression.MSZIP // Count the number of codes for each code length int[] bl_count = new int[max_bits + 1]; - for (int i = 0; i < num_codes; i++) + for (int i = 0; i < numCodes; i++) { int length = lengths[i]; bl_count[length]++; @@ -45,8 +45,8 @@ namespace SabreTools.Compression.MSZIP // values for all codes of the same length with the base // values determined at step 2. Codes that are never used // (which have a bit length of zero) must not be assigned a value. - int[] tree = new int[num_codes]; - for (int i = 0; i < num_codes; i++) + int[] tree = new int[numCodes]; + for (int i = 0; i < numCodes; i++) { byte len = lengths[i]; if (len == 0) @@ -58,7 +58,7 @@ namespace SabreTools.Compression.MSZIP } // Now insert the values into the structure - for (int i = 0; i < num_codes; i++) + for (int i = 0; i < numCodes; i++) { // If we have a 0-length code byte len = lengths[i];