SabreTools/SabreTools.Compression
1
0
Fork 0
mirror of https://github.com/SabreTools/SabreTools.Compression.git synced 2026-09-22 06:45:04 +00:00
Code Issues Packages Projects Releases 19 Wiki Activity

Add RFC1951 implementation

Browse Source
This commit is contained in:
Matt Nadareski
2023-09-22 11:27:07 -04:00
parent bd9258d9fa
commit 297fffe8d7
4 changed files with 355 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
BitStream.cs
View File

@@ -126,23 +126,18 @@ namespace SabreTools.Compression
/// Read a byte, if possible
/// </summary>
/// <returns>The next byte, null on error or end of stream</returns>
/// <remarks>Assumes the stream is byte-aligned</remarks>
public byte? ReadByte()
{
// If we don't have a value cached
if (_bitBuffer == null)
try
{
try
{
return _source.ReadByteValue();
}
catch
{
return null;
}
Discard();
return _source.ReadByteValue();
}
catch
{
return null;
}
// Otherwise, assemble the value from the next bits
throw new NotImplementedException();
}
/// <summary>
@@ -154,6 +149,7 @@ namespace SabreTools.Compression
{
try
{
Discard();
return _source.ReadUInt16();
}
catch
@@ -171,6 +167,7 @@ namespace SabreTools.Compression
{
try
{
Discard();
return _source.ReadUInt32();
}
catch
@@ -188,6 +185,7 @@ namespace SabreTools.Compression
{
try
{
Discard();
return _source.ReadUInt64();
}
catch
@@ -206,6 +204,7 @@ namespace SabreTools.Compression
{
try
{
Discard();
return _source.ReadBytes(bytes);
}
catch

53
MSZIP/Constants.cs Normal file
View File

@@ -0,0 +1,53 @@
namespace SabreTools.Compression.MSZIP
{
public static class Constants
{
/// <summary>
/// Alphabet for fixed Huffman encoding
/// </summary>
public static readonly byte[] FixedAlphabet = new byte[19]
{
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15,
};
/// <summary>
/// Extra bits for length codes 257-285
/// </summary>
public static readonly byte[] MatchExtraBits = new byte[29]
{
0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 0,
};
/// <summary>
/// Initial lengths for length codes 257-285
/// </summary>
public static readonly ushort[] MatchLengths = new ushort[29]
{
3, 4, 5, 6, 7, 8, 9, 10, 11, 13,
15, 17, 19, 23, 27, 31, 35, 43, 51, 59,
67, 83, 99, 115, 131, 163, 195, 227, 258,
};
/// <summary>
/// Extra bits for distance codes 0-29
/// </summary>
public static readonly byte[] DistanceExtraBits = new byte[30]
{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
};
/// <summary>
/// Initial lengths for distance codes 0-29
/// </summary>
public static readonly ushort[] DistanceLengths = new ushort[30]
{
1, 2, 3, 4, 5, 7, 9, 13, 17, 25,
33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577
};
}
}

343
MSZIP/Decompressor.cs
View File

@@ -1,29 +1,43 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Models.Compression.MSZIP;
using static SabreTools.Compression.MSZIP.Constants;
namespace SabreTools.Compression.MSZIP
{
// TODO: Combine parts of the fixed and dynamic implementation
/// <see href="https://www.rfc-editor.org/rfc/rfc1951"/>
public class Decompressor
{
/// <summary>
/// Decompress a stream into a byte array
/// Internal bitstream to use for decompression
/// </summary>
private BitStream _bitStream;
/// <summary>
/// Create a new Decompressor from a Stream
/// </summary>
/// <param name="input">Stream to decompress</param>
/// <returns>Byte array containing the decompressed data on success, null on error</returns>
/// <see href="https://www.rfc-editor.org/rfc/rfc1951"/>
public static byte[] Decompress(Stream input)
public Decompressor(Stream input)
{
// If we have an invalid stream
if (input == null || !input.CanRead || !input.CanSeek)
return null;
throw new ArgumentException(nameof(input));
// Wrap the stream in a BitStream
var bitStream = new BitStream(input);
_bitStream = new BitStream(input);
}
/// <summary>
/// Decompress a stream into a byte array
/// </summary>
/// <returns>Byte array containing the decompressed data on success, null on error</returns>
public byte[] Process()
{
// Try to read the header
var blockHeader = ReadBlockHeader(bitStream);
var blockHeader = ReadBlockHeader();
if (blockHeader.Signature != 0x4B43)
return null;
@@ -32,79 +46,61 @@ namespace SabreTools.Compression.MSZIP
while (true)
{
// Try to read the deflate block header
var deflateBlockHeader = ReadDeflateBlockHeader(bitStream);
if (deflateBlockHeader.BTYPE == CompressionType.Reserved)
break;
// If stored with no compression
if (deflateBlockHeader.BTYPE == CompressionType.NoCompression)
var deflateBlockHeader = ReadDeflateBlockHeader();
switch (deflateBlockHeader.BTYPE)
{
// Skip any remaining bits in current partially processed byte
bitStream.Discard();
// Read LEN and NLEN
var nonCompressedBlockHeader = ReadNonCompressedBlockHeader(bitStream);
if (nonCompressedBlockHeader.LEN == 0 && nonCompressedBlockHeader.NLEN == 0)
// If stored with no compression
case CompressionType.NoCompression:
byte[] bytes00 = ReadNoCompression();
bytes.AddRange(bytes00);
break;
// Copy LEN bytes of data to output
byte[] uncompressed = bitStream.ReadBytes(nonCompressedBlockHeader.LEN);
bytes.AddRange(uncompressed);
}
// If compressed with fixed Huffman codes
case CompressionType.FixedHuffman:
byte[] bytes01 = ReadFixedHuffman();
bytes.AddRange(bytes01);
break;
// Otherwise
else
{
// If compressed with dynamic Huffman codes
if (deflateBlockHeader.BTYPE == CompressionType.DynamicHuffman)
{
// Read representation of code trees
}
case CompressionType.DynamicHuffman:
byte[] bytes10 = ReadDynamicHuffman();
bytes.AddRange(bytes10);
break;
// Loop (until end of block code recognized)
while (true)
{
/*
decode literal/length value from input stream
if value < 256
copy value (literal byte) to output stream
otherwise
if value = end of block (256)
break from loop
otherwise (value = 257..285)
decode distance from input stream
move backwards distance bytes in the output
stream, and copy length bytes from this
position to the output stream.
*/
}
// Reserved is not allowed and is treated as an error
case CompressionType.Reserved:
default:
return null;
}
// If we're at the final block, exit out of the loop
if (deflateBlockHeader.BFINAL)
break;
}
throw new NotImplementedException();
return bytes.ToArray();
}
#region Headers
/// <summary>
/// Read a BlockHeader from the input stream
/// </summary>
private static BlockHeader ReadBlockHeader(BitStream input)
private BlockHeader ReadBlockHeader()
{
var header = new BlockHeader();
header.Signature = input.ReadUInt16() ?? 0;
header.Signature = _bitStream.ReadUInt16() ?? 0;
return header;
}
/// <summary>
/// Read a DeflateBlockHeader from the input stream
/// </summary>
private static DeflateBlockHeader ReadDeflateBlockHeader(BitStream input)
private DeflateBlockHeader ReadDeflateBlockHeader()
{
var header = new DeflateBlockHeader();
header.BFINAL = input.ReadBit() != 0x01;
byte btype = input.ReadBit() ?? 0x01;
btype <<= 1;
btype |= input.ReadBit() ?? 0x01;
header.BFINAL = _bitStream.ReadBit() != 0x01;
uint? btype = _bitStream.ReadBitsLSB(2) ?? 0b11;
header.BTYPE = (CompressionType)btype;
return header;
}
@@ -112,12 +108,241 @@ namespace SabreTools.Compression.MSZIP
/// <summary>
/// Read a NonCompressedBlockHeader from the input stream
/// </summary>
private static NonCompressedBlockHeader ReadNonCompressedBlockHeader(BitStream input)
private NonCompressedBlockHeader ReadNonCompressedBlockHeader()
{
var header = new NonCompressedBlockHeader();
header.LEN = input.ReadUInt16() ?? 0;
header.NLEN = input.ReadUInt16() ?? 0;
header.LEN = _bitStream.ReadUInt16() ?? 0;
header.NLEN = _bitStream.ReadUInt16() ?? 0;
return header;
}
#endregion
#region Data
/// <summary>
/// Read an RFC1951 block with no compression
/// </summary>
private byte[] ReadNoCompression()
{
// Skip any remaining bits in current partially processed byte
_bitStream.Discard();
// Read LEN and NLEN
var nonCompressedBlockHeader = ReadNonCompressedBlockHeader();
if (nonCompressedBlockHeader.LEN == 0 && nonCompressedBlockHeader.NLEN == 0)
return null;
// Copy LEN bytes of data to output
return _bitStream.ReadBytes(nonCompressedBlockHeader.LEN);
}
/// <summary>
/// Read an RFC1951 block with fixed Huffman compression
/// </summary>
private byte[] ReadFixedHuffman()
{
var bytes = new List<byte>();
while (true)
{
// Read the next symbol from the bitstream
uint? sym = _bitStream.ReadBitsMSB(7);
if (sym == null)
return null;
// If we have a symbol number <= 23, adjust it
if (sym <= 23)
{
sym += 256;
}
// Read the next bit otherwise
else
{
sym = (sym << 1) + _bitStream.ReadBit();
if (sym <= 191)
sym -= 48;
else if (sym <= 199)
sym += 88;
else
sym = (sym << 1) + _bitStream.ReadBit() - 256;
}
// If we have an immediate symbol
if (sym < 256)
{
bytes.Add((byte)sym);
}
// If we have the ending symbol
else if (sym == 256)
{
break;
}
// If we have a length/distance pair
else
{
sym -= 257;
uint? length = MatchLengths[(int)sym] + _bitStream.ReadBitsLSB(MatchExtraBits[(int)sym]);
if (length == null)
return null;
uint? distanceCode = _bitStream.ReadBitsMSB(5);
if (distanceCode == null)
return null;
uint? distance = DistanceLengths[(int)distanceCode] + _bitStream.ReadBitsLSB(DistanceExtraBits[(int)distanceCode]);
if (distance == null)
return null;
byte[] arr = bytes.Skip(bytes.Count - (int)distance).Take((int)length).ToArray();
bytes.AddRange(arr);
}
}
return bytes.ToArray();
}
/// <summary>
/// Read an RFC1951 block with dynamic Huffman compression
/// </summary>
private byte[] ReadDynamicHuffman()
{
// Setup the counts first
uint numLiteral = 257 + _bitStream.ReadBitsLSB(5) ?? 0;
uint numDistance = 1 + _bitStream.ReadBitsLSB(5) ?? 0;
uint numLength = 4 + _bitStream.ReadBitsLSB(4) ?? 0;
// Convert the alphabet based on lengths
byte[] lengthLengths = new byte[19];
for (int i = 0; i < numLength; i++)
{
lengthLengths[FixedAlphabet[i]] = (byte)_bitStream.ReadBitsLSB(3);
}
for (int i = (int)numLength; i < 19; i++)
{
lengthLengths[FixedAlphabet[i]] = 0;
}
// Make the lengths tree
HuffmanDecoder lengthTree = new HuffmanDecoder(lengthLengths, 19);
// Setup the literal and distance lengths
byte[] literalLengths = new byte[288];
byte[] distanceLengths = new byte[32];
// Read the literal and distance codes
int repeatCode = 1;
uint leftover = ReadHuffmanLengths(lengthTree, literalLengths, numLiteral, 0, ref repeatCode);
_ = ReadHuffmanLengths(lengthTree, distanceLengths, numDistance, leftover, ref repeatCode);
// Make the literal and distance trees
HuffmanDecoder literalTree = new HuffmanDecoder(literalLengths, numLiteral);
HuffmanDecoder distanceTree = new HuffmanDecoder(distanceLengths, numDistance);
// Now loop and decode
var bytes = new List<byte>();
while (true)
{
// Decode the next literal value
int sym = literalTree.Decode(_bitStream);
// If we have an immediate symbol
if (sym < 256)
{
bytes.Add((byte)sym);
}
// If we have the ending symbol
else if (sym == 256)
{
break;
}
// If we have a length/distance pair
else
{
sym -= 257;
uint? length = MatchLengths[sym] + _bitStream.ReadBitsLSB(MatchExtraBits[sym]);
if (length == null)
return null;
int distanceCode = distanceTree.Decode(_bitStream);
uint? distance = DistanceLengths[distanceCode] + _bitStream.ReadBitsLSB(DistanceExtraBits[distanceCode]);
if (distance == null)
return null;
byte[] arr = bytes.Skip(bytes.Count - (int)distance).Take((int)length).ToArray();
bytes.AddRange(arr);
}
}
// Return the decoded array
return bytes.ToArray();
}
/// <summary>
/// Read the huffman lengths
/// </summary>
private uint ReadHuffmanLengths(HuffmanDecoder lengthTree, byte[] lengths, uint numCodes, uint repeat, ref int repeatCode)
{
int i = 0;
// First fill in any repeat codes
while (repeat > 0)
{
lengths[i++] = (byte)repeatCode;
repeat--;
}
// Then process the rest of the table
while (i < numCodes)
{
// Get the next length encoding from the stream
int lengthEncoding = lengthTree.Decode(_bitStream);
// Values less than 16 are encoded directly
if (lengthEncoding < 16)
{
lengths[i++] = (byte)lengthEncoding;
repeatCode = lengthEncoding;
}
// Otherwise, the repeat count is based on the next values
else
{
// Determine the repeat count and code from the encoding
if (lengthEncoding == 16)
{
repeat = 3 + _bitStream.ReadBitsLSB(2) ?? 0;
}
else if (lengthEncoding == 17)
{
repeat = 3 + _bitStream.ReadBitsLSB(3) ?? 0;
repeatCode = 0;
}
else if (lengthEncoding == 18)
{
repeat = 11 + _bitStream.ReadBitsLSB(7) ?? 0;
repeatCode = 0;
}
// Read in the expected lengths
while (i < numCodes && repeat > 0)
{
lengths[i++] = (byte)repeatCode;
repeat--;
}
}
}
// Return any repeat value we have left over
return repeat;
}
#endregion
}
}

12
MSZIP/HuffmanDecoder.cs
View File

@@ -14,8 +14,8 @@ namespace SabreTools.Compression.MSZIP
/// Create a Huffman tree to decode with
/// </summary>
/// <param name="lengths">Array representing the number of bits for each value</param>
/// <param name="num_codes">Number of Huffman codes encoded</param>
public HuffmanDecoder(byte[] lengths, uint num_codes)
/// <param name="numCodes">Number of Huffman codes encoded</param>
public HuffmanDecoder(byte[] lengths, uint numCodes)
{
// Set the root to null for now
_root = null;
@@ -25,7 +25,7 @@ namespace SabreTools.Compression.MSZIP
// Count the number of codes for each code length
int[] bl_count = new int[max_bits + 1];
for (int i = 0; i < num_codes; i++)
for (int i = 0; i < numCodes; i++)
{
int length = lengths[i];
bl_count[length]++;
@@ -45,8 +45,8 @@ namespace SabreTools.Compression.MSZIP
// values for all codes of the same length with the base
// values determined at step 2. Codes that are never used
// (which have a bit length of zero) must not be assigned a value.
int[] tree = new int[num_codes];
for (int i = 0; i < num_codes; i++)
int[] tree = new int[numCodes];
for (int i = 0; i < numCodes; i++)
{
byte len = lengths[i];
if (len == 0)
@@ -58,7 +58,7 @@ namespace SabreTools.Compression.MSZIP
}
// Now insert the values into the structure
for (int i = 0; i < num_codes; i++)
for (int i = 0; i < numCodes; i++)
{
// If we have a 0-length code
byte len = lengths[i];