diff --git a/LZ/Decompressor.cs b/LZ/Decompressor.cs index f8c30b5..7194cd2 100644 --- a/LZ/Decompressor.cs +++ b/LZ/Decompressor.cs @@ -5,7 +5,7 @@ using SabreTools.IO; using SabreTools.Models.Compression.LZ; using static SabreTools.Models.Compression.LZ.Constants; -namespace BinaryObjectScanner.Compression.LZ +namespace SabreTools.Compression.LZ { /// public class Decompressor diff --git a/LZX/Bits.cs b/LZX/Bits.cs new file mode 100644 index 0000000..e8a7a02 --- /dev/null +++ b/LZX/Bits.cs @@ -0,0 +1,12 @@ +namespace SabreTools.Compression.LZX +{ + /// + public class Bits + { + public uint BitBuffer; + + public int BitsLeft; + + public int InputPosition; //byte* + } +} \ No newline at end of file diff --git a/LZX/Decompressor.cs b/LZX/Decompressor.cs new file mode 100644 index 0000000..9fd2e4e --- /dev/null +++ b/LZX/Decompressor.cs @@ -0,0 +1,759 @@ +using System; +using SabreTools.Compression.LZX; +using static SabreTools.Models.Compression.LZX.Constants; +using static SabreTools.Models.MicrosoftCabinet.Constants; + +namespace SabreTools.Compression.LZX +{ + /// + public class Decompressor + { + /// + /// Initialize an LZX decompressor state + /// + public static bool Init(int window, State state) + { + uint wndsize = (uint)(1 << window); + int posn_slots; + + /* LZX supports window sizes of 2^15 (32Kb) through 2^21 (2Mb) */ + /* if a previously allocated window is big enough, keep it */ + if (window < 15 || window > 21) + return false; + + if (state.actual_size < wndsize) + state.window = null; + + if (state.window == null) + { + state.window = new byte[wndsize]; + state.actual_size = wndsize; + } + + state.window_size = wndsize; + + /* calculate required position slots */ + if (window == 20) posn_slots = 42; + else if (window == 21) posn_slots = 50; + else posn_slots = window << 1; + + /*posn_slots=i=0; while (i < wndsize) i += 1 << CAB(extra_bits)[posn_slots++]; */ + + state.R0 = state.R1 = state.R2 = 1; + state.main_elements = (ushort)(LZX_NUM_CHARS + (posn_slots << 3)); + state.header_read = 0; + state.frames_read = 0; + state.block_remaining = 0; + state.block_type = LZX_BLOCKTYPE_INVALID; + state.intel_curpos = 0; + state.intel_started = 0; + state.window_posn = 0; + + /* initialize tables to 0 (because deltas will be applied to them) */ + // memset(state.MAINTREE_len, 0, sizeof(state.MAINTREE_len)); + // memset(state.LENGTH_len, 0, sizeof(state.LENGTH_len)); + + return true; + } + + /// + /// Decompress a byte array using a given State + /// + public static bool Decompress(State state, int inlen, byte[] inbuf, int outlen, byte[] outbuf) + { + int inpos = 0; // inbuf[0]; + int endinp = inpos + inlen; + int window = 0; // state.window[0]; + int runsrc, rundest; // byte* + + uint window_posn = state.window_posn; + uint window_size = state.window_size; + uint R0 = state.R0; + uint R1 = state.R1; + uint R2 = state.R2; + + uint match_offset, i, j, k; /* ijk used in READ_HUFFSYM macro */ + Bits lb = new Bits(); /* used in READ_LENGTHS macro */ + + int togo = outlen, this_run, main_element, aligned_bits; + int match_length, copy_length, length_footer, extra, verbatim_bits; + + INIT_BITSTREAM(out int bitsleft, out uint bitbuf); + + /* read header if necessary */ + if (state.header_read == 0) + { + i = j = 0; + k = READ_BITS(1, inbuf, ref inpos, ref bitsleft, ref bitbuf); + if (k != 0) + { + i = READ_BITS(16, inbuf, ref inpos, ref bitsleft, ref bitbuf); + j = READ_BITS(16, inbuf, ref inpos, ref bitsleft, ref bitbuf); + } + + state.intel_filesize = (int)((i << 16) | j); /* or 0 if not encoded */ + state.header_read = 1; + } + + /* main decoding loop */ + while (togo > 0) + { + /* last block finished, new block expected */ + if (state.block_remaining == 0) + { + if (state.block_type == LZX_BLOCKTYPE_UNCOMPRESSED) + { + if ((state.block_length & 1) != 0) + inpos++; /* realign bitstream to word */ + + INIT_BITSTREAM(out bitsleft, out bitbuf); + } + + state.block_type = (ushort)READ_BITS(3, inbuf, ref inpos, ref bitsleft, ref bitbuf); + i = READ_BITS(16, inbuf, ref inpos, ref bitsleft, ref bitbuf); + j = READ_BITS(8, inbuf, ref inpos, ref bitsleft, ref bitbuf); + state.block_remaining = state.block_length = (i << 8) | j; + + switch (state.block_type) + { + case LZX_BLOCKTYPE_ALIGNED: + for (i = 0; i < 8; i++) + { + j = READ_BITS(3, inbuf, ref inpos, ref bitsleft, ref bitbuf); + state.tblALIGNED_len[i] = (byte)j; + } + + make_decode_table(LZX_ALIGNED_MAXSYMBOLS, LZX_ALIGNED_TABLEBITS, state.tblALIGNED_len, state.tblALIGNED_table); + + /* rest of aligned header is same as verbatim */ + goto case LZX_BLOCKTYPE_VERBATIM; + + case LZX_BLOCKTYPE_VERBATIM: + READ_LENGTHS(state.tblMAINTREE_len, 0, 256, lb, state, inbuf, ref inpos, ref bitsleft, ref bitbuf); + READ_LENGTHS(state.tblMAINTREE_len, 256, state.main_elements, lb, state, inbuf, ref inpos, ref bitsleft, ref bitbuf); + make_decode_table(LZX_MAINTREE_MAXSYMBOLS, LZX_MAINTREE_TABLEBITS, state.tblMAINTREE_len, state.tblMAINTREE_table); + if (state.tblMAINTREE_len[0xE8] != 0) + state.intel_started = 1; + + READ_LENGTHS(state.tblLENGTH_len, 0, LZX_NUM_SECONDARY_LENGTHS, lb, state, inbuf, ref inpos, ref bitsleft, ref bitbuf); + make_decode_table(LZX_LENGTH_MAXSYMBOLS, LZX_LENGTH_TABLEBITS, state.tblLENGTH_len, state.tblLENGTH_table); + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + state.intel_started = 1; /* because we can't assume otherwise */ + ENSURE_BITS(16, inbuf, ref inpos, ref bitsleft, ref bitbuf); /* get up to 16 pad bits into the buffer */ + + /* and align the bitstream! */ + if (bitsleft > 16) + inpos -= 2; + + R0 = (uint)(inbuf[inpos + 0] | (inbuf[inpos + 1] << 8) | (inbuf[inpos + 2] << 16) | (inbuf[inpos + 3] << 24)); inpos += 4; + R1 = (uint)(inbuf[inpos + 0] | (inbuf[inpos + 1] << 8) | (inbuf[inpos + 2] << 16) | (inbuf[inpos + 3] << 24)); inpos += 4; + R2 = (uint)(inbuf[inpos + 0] | (inbuf[inpos + 1] << 8) | (inbuf[inpos + 2] << 16) | (inbuf[inpos + 3] << 24)); inpos += 4; + break; + + default: + return false; + } + } + + /* buffer exhaustion check */ + if (inpos > endinp) + { + /* it's possible to have a file where the next run is less than + * 16 bits in size. In this case, the READ_HUFFSYM() macro used + * in building the tables will exhaust the buffer, so we should + * allow for this, but not allow those accidentally read bits to + * be used (so we check that there are at least 16 bits + * remaining - in this boundary case they aren't really part of + * the compressed data) + */ + if (inpos > (endinp + 2) || bitsleft < 16) + return false; + } + + while ((this_run = (int)state.block_remaining) > 0 && togo > 0) + { + if (this_run > togo) this_run = togo; + togo -= this_run; + state.block_remaining -= (uint)this_run; + + /* apply 2^x-1 mask */ + window_posn &= window_size - 1; + + /* runs can't straddle the window wraparound */ + if ((window_posn + this_run) > window_size) + return false; + + switch (state.block_type) + { + + case LZX_BLOCKTYPE_VERBATIM: + while (this_run > 0) + { + main_element = READ_HUFFSYM(state.tblMAINTREE_table, state.tblMAINTREE_len, LZX_MAINTREE_TABLEBITS, LZX_MAINTREE_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + if (main_element < LZX_NUM_CHARS) + { + /* literal: 0 to LZX_NUM_CHARS-1 */ + state.window[window + window_posn++] = (byte)main_element; + this_run--; + } + else + { + /* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */ + main_element -= LZX_NUM_CHARS; + + match_length = main_element & LZX_NUM_PRIMARY_LENGTHS; + if (match_length == LZX_NUM_PRIMARY_LENGTHS) + { + length_footer = READ_HUFFSYM(state.tblLENGTH_table, state.tblLENGTH_len, LZX_LENGTH_TABLEBITS, LZX_LENGTH_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_length += length_footer; + } + + match_length += LZX_MIN_MATCH; + match_offset = (uint)(main_element >> 3); + + if (match_offset > 2) + { + /* not repeated offset */ + if (match_offset != 3) + { + extra = state.ExtraBits[match_offset]; + verbatim_bits = (int)READ_BITS(extra, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_offset = (uint)(state.PositionSlotBases[match_offset] - 2 + verbatim_bits); + } + else + { + match_offset = 1; + } + + /* update repeated offset LRU queue */ + R2 = R1; R1 = R0; R0 = match_offset; + } + else if (match_offset == 0) + { + match_offset = R0; + } + else if (match_offset == 1) + { + match_offset = R1; + R1 = R0; R0 = match_offset; + } + else /* match_offset == 2 */ + { + match_offset = R2; + R2 = R0; R0 = match_offset; + } + + rundest = (int)(window + window_posn); + this_run -= match_length; + + /* copy any wrapped around source data */ + if (window_posn >= match_offset) + { + /* no wrap */ + runsrc = (int)(rundest - match_offset); + } + else + { + runsrc = (int)(rundest + (window_size - match_offset)); + copy_length = (int)(match_offset - window_posn); + if (copy_length < match_length) + { + match_length -= copy_length; + window_posn += (uint)copy_length; + while (copy_length-- > 0) + { + state.window[rundest++] = state.window[runsrc++]; + } + + runsrc = window; + } + } + + window_posn += (uint)match_length; + + /* copy match data - no worries about destination wraps */ + while (match_length-- > 0) + { + state.window[rundest++] = state.window[runsrc++]; + } + } + } + break; + + case LZX_BLOCKTYPE_ALIGNED: + while (this_run > 0) + { + main_element = READ_HUFFSYM(state.tblMAINTREE_table, state.tblMAINTREE_len, LZX_MAINTREE_TABLEBITS, LZX_MAINTREE_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + + if (main_element < LZX_NUM_CHARS) + { + /* literal: 0 to LZX_NUM_CHARS-1 */ + state.window[window + window_posn++] = (byte)main_element; + this_run--; + } + else + { + /* mverbatim_bitsatch: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */ + main_element -= LZX_NUM_CHARS; + + match_length = main_element & LZX_NUM_PRIMARY_LENGTHS; + if (match_length == LZX_NUM_PRIMARY_LENGTHS) + { + length_footer = READ_HUFFSYM(state.tblLENGTH_table, state.tblLENGTH_len, LZX_LENGTH_TABLEBITS, LZX_LENGTH_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_length += length_footer; + } + match_length += LZX_MIN_MATCH; + + match_offset = (uint)(main_element >> 3); + + if (match_offset > 2) + { + /* not repeated offset */ + extra = state.ExtraBits[match_offset]; + match_offset = state.PositionSlotBases[match_offset] - 2; + if (extra > 3) + { + /* verbatim and aligned bits */ + extra -= 3; + verbatim_bits = (int)READ_BITS(extra, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_offset += (uint)(verbatim_bits << 3); + aligned_bits = READ_HUFFSYM(state.tblALIGNED_table, state.tblALIGNED_len, LZX_ALIGNED_TABLEBITS, LZX_ALIGNED_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_offset += (uint)aligned_bits; + } + else if (extra == 3) + { + /* aligned bits only */ + aligned_bits = READ_HUFFSYM(state.tblALIGNED_table, state.tblALIGNED_len, LZX_ALIGNED_TABLEBITS, LZX_ALIGNED_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_offset += (uint)aligned_bits; + } + else if (extra > 0) + { + /* extra==1, extra==2 */ + /* verbatim bits only */ + verbatim_bits = (int)READ_BITS(extra, inbuf, ref inpos, ref bitsleft, ref bitbuf); + match_offset += (uint)verbatim_bits; + } + else /* extra == 0 */ + { + /* ??? */ + match_offset = 1; + } + + /* update repeated offset LRU queue */ + R2 = R1; R1 = R0; R0 = match_offset; + } + else if (match_offset == 0) + { + match_offset = R0; + } + else if (match_offset == 1) + { + match_offset = R1; + R1 = R0; R0 = match_offset; + } + else /* match_offset == 2 */ + { + match_offset = R2; + R2 = R0; R0 = match_offset; + } + + rundest = (int)(window + window_posn); + this_run -= match_length; + + /* copy any wrapped around source data */ + if (window_posn >= match_offset) + { + /* no wrap */ + runsrc = (int)(rundest - match_offset); + } + else + { + runsrc = (int)(rundest + (window_size - match_offset)); + copy_length = (int)(match_offset - window_posn); + if (copy_length < match_length) + { + match_length -= copy_length; + window_posn += (uint)copy_length; + while (copy_length-- > 0) + { + state.window[rundest++] = state.window[runsrc++]; + } + + runsrc = window; + } + } + + window_posn += (uint)match_length; + + /* copy match data - no worries about destination wraps */ + while (match_length-- > 0) + { + state.window[rundest++] = state.window[runsrc++]; + } + } + } + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + if ((inpos + this_run) > endinp) + return false; + + Array.Copy(inbuf, inpos, state.window, window + window_posn, this_run); + inpos += this_run; + window_posn += (uint)this_run; + break; + + default: + return false; /* might as well */ + } + + } + } + + if (togo != 0) + return false; + + Array.Copy(state.window, window + ((window_posn == 0) ? window_size : window_posn) - outlen, outbuf, 0, outlen); + + state.window_posn = window_posn; + state.R0 = R0; + state.R1 = R1; + state.R2 = R2; + + /* intel E8 decoding */ + if ((state.frames_read++ < 32768) && state.intel_filesize != 0) + { + if (outlen <= 6 || state.intel_started == 0) + { + state.intel_curpos += outlen; + } + else + { + int data = 0; // outbuf[0]; + int dataend = data + outlen - 10; + int curpos = state.intel_curpos; + int filesize = state.intel_filesize; + int abs_off, rel_off; + + state.intel_curpos = curpos + outlen; + + while (data < dataend) + { + if (outbuf[data++] != 0xE8) + { + curpos++; + continue; + } + + abs_off = outbuf[data + 0] | (outbuf[data + 1] << 8) | (outbuf[data + 2] << 16) | (outbuf[data + 3] << 24); + if ((abs_off >= -curpos) && (abs_off < filesize)) + { + rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize; + outbuf[data + 0] = (byte)rel_off; + outbuf[data + 1] = (byte)(rel_off >> 8); + outbuf[data + 2] = (byte)(rel_off >> 16); + outbuf[data + 3] = (byte)(rel_off >> 24); + } + data += 4; + curpos += 5; + } + } + } + + return true; + } + /// + /// Read and build the Huffman tree from the lengths + /// + private static int ReadLengths(byte[] lengths, uint first, uint last, Bits lb, State state, byte[] inbuf) + { + uint x, y; + uint bitbuf = lb.BitBuffer; + int bitsleft = lb.BitsLeft; + int inpos = lb.InputPosition; + + for (x = 0; x < 20; x++) + { + y = READ_BITS(4, inbuf, ref inpos, ref bitsleft, ref bitbuf); + state.tblPRETREE_len[x] = (byte)y; + } + + make_decode_table(LZX_PRETREE_MAXSYMBOLS, LZX_PRETREE_TABLEBITS, state.tblPRETREE_len, state.tblPRETREE_table); + + for (x = first; x < last;) + { + int z = READ_HUFFSYM(state.tblPRETREE_table, state.tblPRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + if (z == 17) + { + y = READ_BITS(4, inbuf, ref inpos, ref bitsleft, ref bitbuf); + y += 4; + while (y-- > 0) + { + lengths[x++] = 0; + } + } + else if (z == 18) + { + y = READ_BITS(5, inbuf, ref inpos, ref bitsleft, ref bitbuf); + y += 20; + while (y-- > 0) + { + lengths[x++] = 0; + } + } + else if (z == 19) + { + y = READ_BITS(1, inbuf, ref inpos, ref bitsleft, ref bitbuf); + y += 4; + + z = READ_HUFFSYM(state.tblPRETREE_table, state.tblPRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS, inbuf, ref inpos, ref bitsleft, ref bitbuf); + z = lengths[x] - z; + if (z < 0) + z += 17; + + while (y-- > 0) + { + lengths[x++] = (byte)z; + } + } + else + { + z = lengths[x] - z; + if (z < 0) + z += 17; + + lengths[x++] = (byte)z; + } + } + + lb.BitBuffer = bitbuf; + lb.BitsLeft = bitsleft; + lb.InputPosition = inpos; + return 0; + } + + // Bitstream reading macros (LZX / intel little-endian byte order) + #region Bitstream Reading Macros + + /* + * These bit access routines work by using the area beyond the MSB and the + * LSB as a free source of zeroes. This avoids having to mask any bits. + * So we have to know the bit width of the bitbuffer variable. + */ + + /// + /// Should be used first to set up the system + /// + private static void INIT_BITSTREAM(out int bitsleft, out uint bitbuf) + { + bitsleft = 0; + bitbuf = 0; + } + + /// + /// Ensures there are at least N bits in the bit buffer. It can guarantee + // up to 17 bits (i.e. it can read in 16 new bits when there is down to + /// 1 bit in the buffer, and it can read 32 bits when there are 0 bits in + /// the buffer). + /// + /// Quantum reads bytes in normal order; LZX is little-endian order + private static void ENSURE_BITS(int n, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + while (bitsleft < n) + { + byte b0 = inpos + 0 < inbuf.Length ? inbuf[inpos + 0] : (byte)0; + byte b1 = inpos + 1 < inbuf.Length ? inbuf[inpos + 1] : (byte)0; + + bitbuf |= (uint)(((b1 << 8) | b0) << (16 - bitsleft)); + bitsleft += 16; + inpos += 2; + } + } + + /// + /// Extracts (without removing) N bits from the bit buffer + /// + private static uint PEEK_BITS(int n, uint bitbuf) + { + return bitbuf >> (32 - n); + } + + /// + /// Removes N bits from the bit buffer + /// + private static void REMOVE_BITS(int n, ref int bitsleft, ref uint bitbuf) + { + bitbuf <<= n; + bitsleft -= n; + } + + /// + /// Takes N bits from the buffer and puts them in v. + /// + private static uint READ_BITS(int n, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + uint v = 0; + if (n > 0) + { + ENSURE_BITS(n, inbuf, ref inpos, ref bitsleft, ref bitbuf); + v = PEEK_BITS(n, bitbuf); + REMOVE_BITS(n, ref bitsleft, ref bitbuf); + } + + return v; + } + + #endregion + + #region Huffman Methods + + /// + /// This function was coded by David Tritscher. It builds a fast huffman + /// decoding table out of just a canonical huffman code lengths table. + /// + /// Total number of symbols in this huffman tree. + /// + /// Any symbols with a code length of nbits or less can be decoded + /// in one lookup of the table. + /// + /// A table to get code lengths from [0 to syms-1] + /// The table to fill up with decoded symbols and pointers. + /// + /// OK: 0 + /// error: 1 + /// + private static int make_decode_table(uint nsyms, uint nbits, byte[] length, ushort[] table) + { + ushort sym; + uint leaf; + byte bit_num = 1; + uint fill; + uint pos = 0; /* the current position in the decode table */ + uint table_mask = (uint)(1 << (int)nbits); + uint bit_mask = table_mask >> 1; /* don't do 0 length codes */ + uint next_symbol = bit_mask; /* base of allocation for long codes */ + + /* fill entries for codes short enough for a direct mapping */ + while (bit_num <= nbits) + { + for (sym = 0; sym < nsyms; sym++) + { + if (length[sym] == bit_num) + { + leaf = pos; + + if ((pos += bit_mask) > table_mask) return 1; /* table overrun */ + + /* fill all possible lookups of this symbol with the symbol itself */ + fill = bit_mask; + while (fill-- > 0) table[leaf++] = sym; + } + } + bit_mask >>= 1; + bit_num++; + } + + /* if there are any codes longer than nbits */ + if (pos != table_mask) + { + /* clear the remainder of the table */ + for (sym = (ushort)pos; sym < table_mask; sym++) table[sym] = 0; + + /* give ourselves room for codes to grow by up to 16 more bits */ + pos <<= 16; + table_mask <<= 16; + bit_mask = 1 << 15; + + while (bit_num <= 16) + { + for (sym = 0; sym < nsyms; sym++) + { + if (length[sym] == bit_num) + { + leaf = pos >> 16; + for (fill = 0; fill < bit_num - nbits; fill++) + { + /* if this path hasn't been taken yet, 'allocate' two entries */ + if (table[leaf] == 0) + { + table[(next_symbol << 1)] = 0; + table[(next_symbol << 1) + 1] = 0; + table[leaf] = (ushort)next_symbol++; + } + /* follow the path and select either left or right for next bit */ + leaf = (uint)(table[leaf] << 1); + if (((pos >> (int)(15 - fill)) & 1) != 0) leaf++; + } + table[leaf] = sym; + + if ((pos += bit_mask) > table_mask) return 1; /* table overflow */ + } + } + bit_mask >>= 1; + bit_num++; + } + } + + /* full table? */ + if (pos == table_mask) return 0; + + /* either erroneous table, or all elements are 0 - let's find out. */ + for (sym = 0; sym < nsyms; sym++) if (length[sym] != 0) return 1; + return 0; + } + + #endregion + + // Huffman macros + #region Huffman Macros + + /// + /// Decodes one huffman symbol from the bitstream using the stated table and + /// puts it in v. + /// + private static int READ_HUFFSYM(ushort[] hufftbl, byte[] lentable, int tablebits, int maxsymbols, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + int v = 0, i, j = 0; + ENSURE_BITS(16, inbuf, ref inpos, ref bitsleft, ref bitbuf); + if ((i = hufftbl[PEEK_BITS(tablebits, bitbuf)]) >= maxsymbols) + { + j = 1 << (32 - tablebits); + do + { + j >>= 1; + i <<= 1; + i |= (bitbuf & j) != 0 ? 1 : 0; + if (j == 0) + throw new System.Exception(); + } while ((i = hufftbl[i]) >= maxsymbols); + } + + j = lentable[v = i]; + REMOVE_BITS(j, ref bitsleft, ref bitbuf); + return v; + } + + /// + /// Reads in code lengths for symbols first to last in the given table. The + /// code lengths are stored in their own special LZX way. + /// + private static bool READ_LENGTHS(byte[] lentable, uint first, uint last, Bits lb, State state, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + lb.BitBuffer = bitbuf; + lb.BitsLeft = bitsleft; + lb.InputPosition = inpos; + + if (ReadLengths(lentable, first, last, lb, state, inbuf) != 0) + return false; + + bitbuf = lb.BitBuffer; + bitsleft = lb.BitsLeft; + inpos = lb.InputPosition; + return true; + } + + #endregion + } +} \ No newline at end of file diff --git a/LZX/State.cs b/LZX/State.cs new file mode 100644 index 0000000..b5839dd --- /dev/null +++ b/LZX/State.cs @@ -0,0 +1,119 @@ +using static SabreTools.Models.Compression.LZX.Constants; + +namespace SabreTools.Compression.LZX +{ + /// + public class State + { + /// + /// the actual decoding window + /// + public byte[] window; + + /// + /// window size (32Kb through 2Mb) + /// + public uint window_size; + + /// + /// window size when it was first allocated + /// + public uint actual_size; + + /// + /// current offset within the window + /// + public uint window_posn; + + /// + /// for the LRU offset system + /// + public uint R0, R1, R2; + + /// + /// number of main tree elements + /// + public ushort main_elements; + + /// + /// have we started decoding at all yet? + /// + public int header_read; + + /// + /// type of this block + /// + public ushort block_type; + + /// + /// uncompressed length of this block + /// + public uint block_length; + + /// + /// uncompressed bytes still left to decode + /// + public uint block_remaining; + + /// + /// the number of CFDATA blocks processed + /// + public uint frames_read; + + /// + /// magic header value used for transform + /// + public int intel_filesize; + + /// + /// current offset in transform space + /// + public int intel_curpos; + + /// + /// have we seen any translatable data yet? + /// + public int intel_started; + + public ushort[] tblPRETREE_table = new ushort[(1 << LZX_PRETREE_TABLEBITS) + (LZX_PRETREE_MAXSYMBOLS << 1)]; + public byte[] tblPRETREE_len = new byte[LZX_PRETREE_MAXSYMBOLS + LZX_LENTABLE_SAFETY]; + + public ushort[] tblMAINTREE_table = new ushort[(1 << LZX_MAINTREE_TABLEBITS) + (LZX_MAINTREE_MAXSYMBOLS << 1)]; + public byte[] tblMAINTREE_len = new byte[LZX_MAINTREE_MAXSYMBOLS + LZX_LENTABLE_SAFETY]; + + public ushort[] tblLENGTH_table = new ushort[(1 << LZX_LENGTH_TABLEBITS) + (LZX_LENGTH_MAXSYMBOLS << 1)]; + public byte[] tblLENGTH_len = new byte[LZX_LENGTH_MAXSYMBOLS + LZX_LENTABLE_SAFETY]; + + public ushort[] tblALIGNED_table = new ushort[(1 << LZX_ALIGNED_TABLEBITS) + (LZX_ALIGNED_MAXSYMBOLS << 1)]; + public byte[] tblALIGNED_len = new byte[LZX_ALIGNED_MAXSYMBOLS + LZX_LENTABLE_SAFETY]; + + #region Decompression Tables + + /// + /// An index to the position slot bases + /// + public uint[] PositionSlotBases = new uint[] + { + 0, 1, 2, 3, 4, 6, 8, 12, + 16, 24, 32, 48, 64, 96, 128, 192, + 256, 384, 512, 768, 1024, 1536, 2048, 3072, + 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, + 65536, 98304, 131072, 196608, 262144, 393216, 524288, 655360, + 786432, 917504, 1048576, 1179648, 1310720, 1441792, 1572864, 1703936, + 1835008, 1966080, 2097152 + }; + + /// + /// How many bits of offset-from-base data is needed + /// + public byte[] ExtraBits = new byte[] + { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, + 15, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17 + }; + + #endregion + } +} \ No newline at end of file diff --git a/MSZIP/Decompressor.cs b/MSZIP/Decompressor.cs new file mode 100644 index 0000000..fe2d1f4 --- /dev/null +++ b/MSZIP/Decompressor.cs @@ -0,0 +1,637 @@ +using System; +using System.Runtime.InteropServices; +using SabreTools.Models.Compression.MSZIP; +using static SabreTools.Models.Compression.MSZIP.Constants; + +namespace SabreTools.Compression.MSZIP +{ + /// + public unsafe class Decompressor + { + /// + /// Decompress a byte array using a given State + /// + public static bool Decompress(State state, int inlen, byte[] inbuf, int outlen, byte[] outbuf) + { + fixed (byte* inpos = inbuf) + { + state.inpos = inpos; + state.bb = state.bk = state.window_posn = 0; + if (outlen > ZIPWSIZE) + return false; + + // CK = Chris Kirmse, official Microsoft purloiner + if (state.inpos[0] != 0x43 || state.inpos[1] != 0x4B) + return false; + + state.inpos += 2; + + int lastBlockFlag = 0; + do + { + if (InflateBlock(&lastBlockFlag, state, inbuf, outbuf) != 0) + return false; + } while (lastBlockFlag == 0); + + // Return success + return true; + } + + } + + /// + /// Decompress a deflated block + /// + private static uint InflateBlock(int* e, State state, byte[] inbuf, byte[] outbuf) + { + // Make local bit buffer + uint b = state.bb; + uint k = state.bk; + + // Read the deflate block header + var header = new DeflateBlockHeader(); + + // Read in last block bit + ZIPNEEDBITS(1, state, ref b, ref k); + header.BFINAL = (*e = (int)b & 1) != 0; + ZIPDUMPBITS(1, ref b, ref k); + + // Read in block type + ZIPNEEDBITS(2, state, ref b, ref k); + header.BTYPE = (CompressionType)(b & 3); + ZIPDUMPBITS(2, ref b, ref k); + + // Restore the global bit buffer + state.bb = b; + state.bk = k; + + // Inflate that block type + switch (header.BTYPE) + { + case CompressionType.NoCompression: + return (uint)DecompressStored(state, inbuf, outbuf); + case CompressionType.FixedHuffman: + return (uint)DecompressFixed(state, inbuf, outbuf); + case CompressionType.DynamicHuffman: + return (uint)DecompressDynamic(state, inbuf, outbuf); + + // Bad block type + case CompressionType.Reserved: + default: + return 2; + } + } + + /// + /// "Decompress" a stored block + /// + private static int DecompressStored(State state, byte[] inbuf, byte[] outbuf) + { + // Make local copies of globals + uint b = state.bb; + uint k = state.bk; + uint w = state.window_posn; + + // Go to byte boundary + int n = (int)(k & 7); + ZIPDUMPBITS(n, ref b, ref k); + + // Read the stored block header + var header = new NonCompressedBlockHeader(); + + // Get the length and its compliment + ZIPNEEDBITS(16, state, ref b, ref k); + header.LEN = (ushort)(b & 0xffff); + ZIPDUMPBITS(16, ref b, ref k); + + ZIPNEEDBITS(16, state, ref b, ref k); + header.NLEN = (ushort)(b & 0xffff); + + if (header.LEN != (~header.NLEN & 0xffff)) + return 1; // Error in compressed data + + ZIPDUMPBITS(16, ref b, ref k); + + // Read and output the compressed data + while (n-- > 0) + { + ZIPNEEDBITS(8, state, ref b, ref k); + outbuf[w++] = (byte)b; + ZIPDUMPBITS(8, ref b, ref k); + } + + // Restore the globals from the locals + state.window_posn = w; + state.bb = b; + state.bk = k; + + return 0; + } + + /// + /// Decompress a block originally compressed with fixed Huffman codes + /// + private static int DecompressFixed(State state, byte[] inbuf, byte[] outbuf) + { + // Create the block header + FixedHuffmanCompressedBlockHeader header = new FixedHuffmanCompressedBlockHeader(); + + fixed (uint* l = state.ll) + fixed (ushort* Zipcplens = CopyLengths) + fixed (ushort* Zipcplext = LiteralExtraBits) + fixed (ushort* Zipcpdist = CopyOffsets) + fixed (ushort* Zipcpdext = DistanceExtraBits) + { + // Assign the literal lengths + state.ll = header.LiteralLengths; + HuffmanNode* fixed_tl; + int fixed_bl = 7; + + // Build the literal length tree + int i = BuildHuffmanTree(l, 288, 257, Zipcplens, Zipcplext, &fixed_tl, &fixed_bl, state); + if (i != 0) + return i; + + // Assign the distance codes + state.ll = header.DistanceCodes; + HuffmanNode* fixed_td; + int fixed_bd = 5; + + // Build the distance code tree + i = BuildHuffmanTree(l, 30, 0, Zipcpdist, Zipcpdext, &fixed_td, &fixed_bd, state); + if (i != 0) + return i; + + // Decompress until an end-of-block code + return InflateCodes(fixed_tl, fixed_td, fixed_bl, fixed_bd, state, inbuf, outbuf); + } + } + + /// + /// Decompress a block originally compressed with dynamic Huffman codes + /// + private static int DecompressDynamic(State state, byte[] inbuf, byte[] outbuf) + { + int i; /* temporary variables */ + uint j; + uint l; /* last length */ + uint m; /* mask for bit lengths table */ + uint n; /* number of lengths to get */ + HuffmanNode* tl; /* literal/length code table */ + HuffmanNode* td; /* distance code table */ + int bl; /* lookup bits for tl */ + int bd; /* lookup bits for td */ + uint nb; /* number of bit length codes */ + uint nl; /* number of literal/length codes */ + uint nd; /* number of distance codes */ + uint b; /* bit buffer */ + uint k; /* number of bits in bit buffer */ + + /* make local bit buffer */ + b = state.bb; + k = state.bk; + + state.ll = new uint[288 + 32]; + fixed (uint* ll = state.ll) + { + /* read in table lengths */ + ZIPNEEDBITS(5, state, ref b, ref k); + nl = 257 + (b & 0x1f); /* number of literal/length codes */ + ZIPDUMPBITS(5, ref b, ref k); + + ZIPNEEDBITS(5, state, ref b, ref k); + nd = 1 + (b & 0x1f); /* number of distance codes */ + ZIPDUMPBITS(5, ref b, ref k); + + ZIPNEEDBITS(4, state, ref b, ref k); + nb = 4 + (b & 0xf); /* number of bit length codes */ + ZIPDUMPBITS(4, ref b, ref k); + if (nl > 288 || nd > 32) + return 1; /* bad lengths */ + + /* read in bit-length-code lengths */ + for (j = 0; j < nb; j++) + { + ZIPNEEDBITS(3, state, ref b, ref k); + state.ll[BitLengthOrder[j]] = b & 7; + ZIPDUMPBITS(3, ref b, ref k); + } + for (; j < 19; j++) + state.ll[BitLengthOrder[j]] = 0; + + /* build decoding table for trees--single level, 7 bit lookup */ + bl = 7; + if ((i = BuildHuffmanTree(ll, 19, 19, null, null, &tl, &bl, state)) != 0) + return i; /* incomplete code set */ + + /* read in literal and distance code lengths */ + n = nl + nd; + m = BitMasks[bl]; + i = (int)(l = 0); + while ((uint)i < n) + { + ZIPNEEDBITS(bl, state, ref b, ref k); + j = (td = tl + (b & m))->b; + ZIPDUMPBITS((int)j, ref b, ref k); + j = td->n; + if (j < 16) /* length of code in bits (0..15) */ + { + state.ll[i++] = l = j; /* save last length in l */ + } + else if (j == 16) /* repeat last length 3 to 6 times */ + { + ZIPNEEDBITS(2, state, ref b, ref k); + j = 3 + (b & 3); + ZIPDUMPBITS(2, ref b, ref k); + if ((uint)i + j > n) + return 1; + while (j-- > 0) + { + state.ll[i++] = l; + } + } + else if (j == 17) /* 3 to 10 zero length codes */ + { + ZIPNEEDBITS(3, state, ref b, ref k); + j = 3 + (b & 7); + ZIPDUMPBITS(3, ref b, ref k); + if ((uint)i + j > n) + return 1; + while (j-- > 0) + state.ll[i++] = 0; + l = 0; + } + else /* j == 18: 11 to 138 zero length codes */ + { + ZIPNEEDBITS(7, state, ref b, ref k); + j = 11 + (b & 0x7f); + ZIPDUMPBITS(7, ref b, ref k); + if ((uint)i + j > n) + return 1; + while (j-- > 0) + state.ll[i++] = 0; + l = 0; + } + } + + /* restore the global bit buffer */ + state.bb = b; + state.bk = k; + + fixed (ushort* Zipcplens = CopyLengths) + fixed (ushort* Zipcplext = LiteralExtraBits) + fixed (ushort* Zipcpdist = CopyOffsets) + fixed (ushort* Zipcpdext = DistanceExtraBits) + { + /* build the decoding tables for literal/length and distance codes */ + bl = ZIPLBITS; + if ((i = BuildHuffmanTree(ll, nl, 257, Zipcplens, Zipcplext, &tl, &bl, state)) != 0) + { + return i; /* incomplete code set */ + } + bd = ZIPDBITS; + BuildHuffmanTree(ll + nl, nd, 0, Zipcpdist, Zipcpdext, &td, &bd, state); + + /* decompress until an end-of-block code */ + if (InflateCodes(tl, td, bl, bd, state, inbuf, outbuf) != 0) + return 1; + + return 0; + } + } + } + + /// + /// Build a Huffman tree from a set of lengths + /// + private static int BuildHuffmanTree(uint* b, uint n, uint s, ushort* d, ushort* e, HuffmanNode** t, int* m, State state) + { + uint a; /* counter for codes of length k */ + uint el; /* length of EOB code (value 256) */ + uint f; /* i repeats in table every f entries */ + int g; /* maximum code length */ + int h; /* table level */ + uint i; /* counter, current code */ + uint j; /* counter */ + int k; /* number of bits in current code */ + int* l; /* stack of bits per table */ + uint* p; /* pointer into state.c[],state.b[],state.v[] */ + HuffmanNode* q; /* points to current table */ + HuffmanNode r = new HuffmanNode(); /* table entry for structure assignment */ + int w; /* bits before this table == (l * h) */ + uint* xp; /* pointer into x */ + int y; /* number of dummy codes added */ + uint z; /* number of entries in current table */ + + fixed (int* state_lx_ptr = state.lx) + { + l = state_lx_ptr + 1; + + /* Generate counts for each bit length */ + el = n > 256 ? b[256] : ZIPBMAX; /* set length of EOB code, if any */ + + for (i = 0; i < ZIPBMAX + 1; ++i) + state.c[i] = 0; + p = b; i = n; + do + { + state.c[*p]++; p++; /* assume all entries <= ZIPBMAX */ + } while (--i > 0); + + if (state.c[0] == n) /* null input--all zero length codes */ + { + *t = null; + *m = 0; + return 0; + } + + /* Find minimum and maximum length, bound *m by those */ + for (j = 1; j <= ZIPBMAX; j++) + { + if (state.c[j] > 0) + break; + } + + k = (int)j; /* minimum code length */ + if ((uint)*m < j) + *m = (int)j; + + for (i = ZIPBMAX; i > 0; i--) + { + if (state.c[i] > 0) + break; + } + + g = (int)i; /* maximum code length */ + if ((uint)*m > i) + *m = (int)i; + + /* Adjust last length count to fill out codes, if needed */ + for (y = 1 << (int)j; j < i; j++, y <<= 1) + { + if ((y -= (int)state.c[j]) < 0) + return 2; /* bad input: more codes than bits */ + } + + if ((y -= (int)state.c[i]) < 0) + return 2; + + state.c[i] += (uint)y; + + /* Generate starting offsets LONGo the value table for each length */ + state.x[1] = j = 0; + + fixed (uint* state_c_ptr = state.c) + fixed (uint* state_x_ptr = state.x) + { + p = state_c_ptr + 1; + xp = state_x_ptr + 2; + while (--i > 0) + { + /* note that i == g from above */ + *xp++ = (j += *p++); + } + } + + /* Make a table of values in order of bit lengths */ + p = b; i = 0; + do + { + if ((j = *p++) != 0) + state.v[state.x[j]++] = i; + } while (++i < n); + + /* Generate the Huffman codes and for each, make the table entries */ + state.x[0] = i = 0; /* first Huffman code is zero */ + + fixed (uint* state_v_ptr = state.v) + { + p = state_v_ptr; /* grab values in bit order */ + h = -1; /* no tables yet--level -1 */ + w = l[-1] = 0; /* no bits decoded yet */ + state.u[0] = default; /* just to keep compilers happy */ + q = null; /* ditto */ + z = 0; /* ditto */ + + /* go through the bit lengths (k already is bits in shortest code) */ + for (; k <= g; k++) + { + a = state.c[k]; + while (a-- > 0) + { + /* here i is the Huffman code of length k bits for value *p */ + /* make tables up to required level */ + while (k > w + l[h]) + { + w += l[h++]; /* add bits already decoded */ + + /* compute minimum size table less than or equal to *m bits */ + if ((z = (uint)(g - w)) > (uint)*m) /* upper limit */ + z = (uint)*m; + + if ((f = (uint)(1 << (int)(j = (uint)(k - w)))) > a + 1) /* try a k-w bit table */ + { /* too few codes for k-w bit table */ + f -= a + 1; /* deduct codes from patterns left */ + fixed (uint* state_c_ptr = state.c) + { + xp = state_c_ptr + k; + while (++j < z) /* try smaller tables up to z bits */ + { + if ((f <<= 1) <= *++xp) + break; /* enough codes to use up j bits */ + f -= *xp; /* else deduct codes from patterns */ + } + } + } + + if ((uint)w + j > el && (uint)w < el) + j = (uint)(el - w); /* make EOB code end at table */ + + z = (uint)(1 << (int)j); /* table entries for j-bit table */ + l[h] = (int)j; /* set table size in stack */ + + /* allocate and link in new table */ + q = (HuffmanNode*)Marshal.AllocHGlobal((int)((z + 1) * sizeof(HuffmanNode))); + *t = q + 1; /* link to list for HuffmanNode_free() */ + *(t = &(*q).t) = null; + state.u[h] = ++q; /* table starts after link */ + + /* connect to last table, if there is one */ + if (h > 0) + { + state.x[h] = i; /* save pattern for backing up */ + r.b = (byte)l[h - 1]; /* bits to dump before this table */ + r.e = (byte)(16 + j); /* bits in this table */ + r.t = q; /* pointer to this table */ + j = (uint)((i & ((1 << w) - 1)) >> (w - l[h - 1])); + state.u[h - 1][j] = r; /* connect to last table */ + } + } + + /* set up table entry in r */ + r.b = (byte)(k - w); + + fixed (uint* state_v_ptr_comp = state.v) + { + if (p >= state_v_ptr_comp + n) + { + r.e = 99; /* out of values--invalid code */ + } + else if (*p < s) + { + r.e = (byte)(*p < 256 ? 16 : 15); /* 256 is end-of-block code */ + r.n = (ushort)*p++; /* simple code is just the value */ + } + else + { + r.e = (byte)e[*p - s]; /* non-simple--look up in lists */ + r.n = d[*p++ - s]; + } + } + + /* fill code-like entries with r */ + f = (uint)(1 << (k - w)); + for (j = i >> w; j < z; j += f) + { + q[j] = r; + } + + /* backwards increment the k-bit code i */ + for (j = (uint)(1 << (k - 1)); (i & j) != 0; j >>= 1) + { + i ^= j; + } + + i ^= j; + + /* backup over finished tables */ + while ((i & ((1 << w) - 1)) != state.x[h]) + w -= l[--h]; /* don't need to update q */ + } + } + } + + /* return actual size of base table */ + *m = l[0]; + } + + /* Return true (1) if we were given an incomplete table */ + return y != 0 && g != 1 ? 1 : 0; + } + + /// + /// Inflate codes into Huffman trees + /// + private static int InflateCodes(HuffmanNode* tl, HuffmanNode* td, int bl, int bd, State state, byte[] inbuf, byte[] outbuf) + { + uint e; /* table entry flag/number of extra bits */ + uint n, d; /* length and index for copy */ + uint w; /* current window position */ + HuffmanNode* t; /* pointer to table entry */ + uint ml, md; /* masks for bl and bd bits */ + uint b; /* bit buffer */ + uint k; /* number of bits in bit buffer */ + + /* make local copies of globals */ + b = state.bb; /* initialize bit buffer */ + k = state.bk; + w = state.window_posn; /* initialize window position */ + + /* inflate the coded data */ + ml = BitMasks[bl]; /* precompute masks for speed */ + md = BitMasks[bd]; + + for (; ; ) + { + ZIPNEEDBITS(bl, state, ref b, ref k); + if ((e = (t = tl + (b & ml))->e) > 16) + { + do + { + if (e == 99) + return 1; + ZIPDUMPBITS(t->b, ref b, ref k); + e -= 16; + ZIPNEEDBITS((int)e, state, ref b, ref k); + } while ((e = (*(t = t->t + (b & BitMasks[e]))).e) > 16); + } + + ZIPDUMPBITS(t->b, ref b, ref k); + if (e == 16) /* then it's a literal */ + { + outbuf[w++] = (byte)t->n; + } + else /* it's an EOB or a length */ + { + /* exit if end of block */ + if (e == 15) + break; + + /* get length of block to copy */ + ZIPNEEDBITS((int)e, state, ref b, ref k); + n = t->n + (b & BitMasks[e]); + ZIPDUMPBITS((int)e, ref b, ref k); + + /* decode distance of block to copy */ + ZIPNEEDBITS(bd, state, ref b, ref k); + + if ((e = (*(t = td + (b & md))).e) > 16) + do + { + if (e == 99) + return 1; + ZIPDUMPBITS(t->b, ref b, ref k); + e -= 16; + ZIPNEEDBITS((int)e, state, ref b, ref k); + } while ((e = (*(t = t->t + (b & BitMasks[e]))).e) > 16); + + ZIPDUMPBITS(t->b, ref b, ref k); + + ZIPNEEDBITS((int)e, state, ref b, ref k); + d = w - t->n - (b & BitMasks[e]); + ZIPDUMPBITS((int)e, ref b, ref k); + + do + { + d &= ZIPWSIZE - 1; + e = ZIPWSIZE - Math.Max(d, w); + e = Math.Min(e, n); + n -= e; + do + { + outbuf[w++] = outbuf[d++]; + } while (--e > 0); + } while (n > 0); + } + } + + /* restore the globals from the locals */ + state.window_posn = w; /* restore global window pointer */ + state.bb = b; /* restore global bit buffer */ + state.bk = k; + + /* done */ + return 0; + } + + #region Macros + + private static void ZIPNEEDBITS(int n, State state, ref uint bitBuffer, ref uint bitCount) + { + while (bitCount < n) + { + int c = *state.inpos++; + bitBuffer |= (uint)(c << (int)bitCount); + bitCount += 8; + } + } + + private static void ZIPDUMPBITS(int n, ref uint bitBuffer, ref uint bitCount) + { + bitBuffer >>= n; + bitCount -= (uint)n; + } + + #endregion + } +} \ No newline at end of file diff --git a/MSZIP/HuffmanNode.cs b/MSZIP/HuffmanNode.cs new file mode 100644 index 0000000..85fa0fd --- /dev/null +++ b/MSZIP/HuffmanNode.cs @@ -0,0 +1,29 @@ +namespace SabreTools.Compression.MSZIP +{ + public unsafe struct HuffmanNode + { + /// + /// Number of extra bits or operation + /// + public byte e; + + /// + /// Number of bits in this code or subcode + /// + public byte b; + + #region v + + /// + /// Literal, length base, or distance base + /// + public ushort n; + + /// + /// Pointer to next level of table + /// + public HuffmanNode* t; + + #endregion + } +} \ No newline at end of file diff --git a/MSZIP/State.cs b/MSZIP/State.cs new file mode 100644 index 0000000..9020ab6 --- /dev/null +++ b/MSZIP/State.cs @@ -0,0 +1,56 @@ +using static SabreTools.Models.Compression.MSZIP.Constants; + +namespace SabreTools.Compression.MSZIP +{ + /// + public unsafe class State + { + /// + /// Current offset within the window + /// + public uint window_posn; + + /// + /// Bit buffer + /// + public uint bb; + + /// + /// Bits in bit buffer + /// + public uint bk; + + /// + /// Literal/length and distance code lengths + /// + public uint[] ll = new uint[288 + 32]; + + /// + /// Bit length count table + /// + public uint[] c = new uint[ZIPBMAX + 1]; + + /// + /// Memory for l[-1..ZIPBMAX-1] + /// + public int[] lx = new int[ZIPBMAX + 1]; + + /// + /// Table stack + /// + public HuffmanNode*[] u = new HuffmanNode*[ZIPBMAX]; + + /// + /// Values in order of bit length + /// + public uint[] v = new uint[ZIPN_MAX]; + + /// + /// Bit offsets, then code stack + /// + public uint[] x = new uint[ZIPBMAX + 1]; + + /// byte* + public byte* inpos; + } +} \ No newline at end of file diff --git a/Quantum/Decompressor.cs b/Quantum/Decompressor.cs new file mode 100644 index 0000000..d36a9ce --- /dev/null +++ b/Quantum/Decompressor.cs @@ -0,0 +1,499 @@ +using System; +using System.Linq; +using SabreTools.Models.Compression.Quantum; +using SabreTools.Models.MicrosoftCabinet; + +namespace SabreTools.Compression.Quantum +{ + /// + /// + /// + /// + public static class Decompressor + { + /// + /// Decompress a byte array using a given State + /// + public static int Decompress(State state, int inlen, byte[] inbuf, int outlen, byte[] outbuf) + { + int inpos = 0, outpos = 0; // inbuf[0], outbuf[0] + int window = 0; // state.Window[0] + int runsrc, rundest; + uint windowPosition = state.WindowPosition; + uint windowSize = state.WindowSize; + + int extra, togo = outlen, matchLength = 0, copyLength; + byte selector, sym; + uint matchOffset = 0; + + // Make local copies of state variables + uint bitBuffer = state.BitBuffer; + int bitsLeft = state.BitsLeft; + + ushort H = 0xFFFF, L = 0; + + // Read initial value of C + ushort C = (ushort)Q_READ_BITS(16, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + + // Apply 2^x-1 mask + windowPosition &= windowSize - 1; + + while (togo > 0) + { + selector = (byte)GET_SYMBOL(state.SelectorModel, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + switch (selector) + { + // Selector 0 = literal model, 64 entries, 0x00-0x3F + case 0: + sym = (byte)GET_SYMBOL(state.Model0, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + state.Window[window + windowPosition++] = sym; + togo--; + break; + + // Selector 1 = literal model, 64 entries, 0x40-0x7F + case 1: + sym = (byte)GET_SYMBOL(state.Model1, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + state.Window[window + windowPosition++] = sym; + togo--; + break; + + // Selector 2 = literal model, 64 entries, 0x80-0xBF + case 2: + sym = (byte)GET_SYMBOL(state.Model2, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + state.Window[window + windowPosition++] = sym; + togo--; + break; + + // Selector 3 = literal model, 64 entries, 0xC0-0xFF + case 3: + sym = (byte)GET_SYMBOL(state.Model3, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + state.Window[window + windowPosition++] = sym; + togo--; + break; + + // Selector 4 = fixed length of 3 + case 4: + sym = (byte)GET_SYMBOL(state.Model4, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + extra = (int)Q_READ_BITS(state.ExtraBits[sym], inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + matchOffset = (uint)(state.PositionSlotBases[sym] + extra + 1); + matchLength = 3; + break; + + // Selector 5 = fixed length of 4 + case 5: + sym = (byte)GET_SYMBOL(state.Model5, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + extra = (int)Q_READ_BITS(state.ExtraBits[sym], inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + matchOffset = (uint)(state.PositionSlotBases[sym] + extra + 1); + matchLength = 4; + break; + + // Selector 6 = variable length + case 6: + sym = (byte)GET_SYMBOL(state.Model6Length, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + extra = (int)Q_READ_BITS(state.LengthExtraBits[sym], inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + matchLength = state.LengthBases[sym] + extra + 5; + + sym = (byte)GET_SYMBOL(state.Model6Position, ref H, ref L, ref C, inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + extra = (int)Q_READ_BITS(state.ExtraBits[sym], inbuf, ref inpos, ref bitsLeft, ref bitBuffer); + matchOffset = (uint)(state.PositionSlotBases[sym] + extra + 1); + break; + + default: + return inpos; + } + + // If this is a match + if (selector >= 4) + { + rundest = (int)(window + windowPosition); + togo -= matchLength; + + // Copy any wrapped around source data + if (windowPosition >= matchOffset) + { + // No wrap + runsrc = (int)(rundest - matchOffset); + } + else + { + runsrc = (int)(rundest + (windowSize - matchOffset)); + copyLength = (int)(matchOffset - windowPosition); + if (copyLength < matchLength) + { + matchLength -= copyLength; + windowPosition += (uint)copyLength; + while (copyLength-- > 0) + { + state.Window[rundest++] = state.Window[rundest++]; + } + + runsrc = window; + } + } + + windowPosition += (uint)matchLength; + + // Copy match data - no worries about destination wraps + while (matchLength-- > 0) + { + state.Window[rundest++] = state.Window[runsrc++]; + + // Handle wraparounds that aren't supposed to happen + if (rundest >= state.Window.Length) + rundest = 0; + if (runsrc >= state.Window.Length) + runsrc = 0; + } + } + + // If we hit the end of the window, copy to the output and wrap + if (windowPosition >= state.Window.Length) + { + Array.Copy(state.Window, 0, outbuf, outpos, Math.Min(windowSize, outlen)); + outpos += (int)Math.Min(windowSize, outlen); + outlen -= (int)Math.Min(windowSize, outlen); + windowPosition = 0; + } + } + + if (togo > 0) + return inpos; + + if (outlen > 0) + { + int sourceIndex = (int)((windowPosition == 0 ? windowSize : windowPosition) - outlen); + Array.Copy(state.Window, sourceIndex, outbuf, outpos, outlen); + } + + // Cache the decompression state variables + state.BitBuffer = bitBuffer; + state.BitsLeft = bitsLeft; + state.WindowPosition = windowPosition; + + return inpos; + } + + /// + /// Initialize a Quantum decompressor state + /// + public static bool InitState(State state, CFFOLDER folder) + { + int window = ((ushort)folder.CompressionType >> 8) & 0x1f; + int level = ((ushort)folder.CompressionType >> 4) & 0xF; + return InitState(state, window, level); + } + + /// + /// Initialize a Quantum decompressor state + /// + public static bool InitState(State state, int window, int level) + { + uint windowSize = (uint)(1 << window); + int maxSize = window * 2; + + // QTM supports window sizes of 2^10 (1Kb) through 2^21 (2Mb) + // If a previously allocated window is big enough, keep it + if (window < 10 || window > 21) + return false; + + // If we don't have the proper window size + if (state.ActualSize < windowSize) + state.Window = null; + + // If we have no window + if (state.Window == null) + { + state.Window = new byte[windowSize]; + state.ActualSize = windowSize; + } + + // Set the window size and position + state.WindowSize = windowSize; + state.WindowPosition = 0; + + // Initialize arithmetic coding models + state.SelectorModel = CreateModel(state.SelectorModelSymbols, 7, 0); + + state.Model0 = CreateModel(state.Model0Symbols, 0x40, 0x00); + state.Model1 = CreateModel(state.Model1Symbols, 0x40, 0x40); + state.Model2 = CreateModel(state.Model2Symbols, 0x40, 0x80); + state.Model3 = CreateModel(state.Model3Symbols, 0x40, 0xC0); + + // Model 4 depends on table size, ranges from 20 to 24 + state.Model4 = CreateModel(state.Model4Symbols, (maxSize < 24) ? maxSize : 24, 0); + + // Model 5 depends on table size, ranges from 20 to 36 + state.Model5 = CreateModel(symbols: state.Model5Symbols, (maxSize < 36) ? maxSize : 36, 0); + + // Model 6 Position depends on table size, ranges from 20 to 42 + state.Model6Position = CreateModel(state.Model6PositionSymbols, (maxSize < 42) ? maxSize : 42, 0); + state.Model6Length = CreateModel(state.Model6LengthSymbols, 27, 0); + + return true; + } + + /// + /// Initialize a Quantum model that decodes symbols from s to (s + n - 1) + /// + private static Model CreateModel(ModelSymbol[] symbols, int entryCount, int initialSymbol) + { + // Set the basic values + Model model = new Model + { + TimeToReorder = 4, + Entries = entryCount, + Symbols = symbols, + }; + + // Clear out the look-up table + model.LookupTable = Enumerable.Repeat(0xFFFF, model.LookupTable.Length).ToArray(); + + // Loop through and build the look-up table + for (ushort i = 0; i < entryCount; i++) + { + // Set up a look-up entry for symbol + model.LookupTable[i + initialSymbol] = i; + + // Create the symbol in the table + model.Symbols[i] = new ModelSymbol + { + Symbol = (ushort)(i + initialSymbol), + CumulativeFrequency = (ushort)(entryCount - i), + }; + } + + // Set the last symbol frequency to 0 + model.Symbols[entryCount] = new ModelSymbol { CumulativeFrequency = 0 }; + return model; + } + + /// + /// Update the Quantum model for a particular symbol + /// + private static void UpdateModel(Model model, int symbol) + { + // Update the cumulative frequency for all symbols less than the provided + for (int i = 0; i < symbol; i++) + { + model.Symbols[i].CumulativeFrequency += 8; + } + + // If the first symbol still has a cumulative frequency under 3800 + if (model.Symbols[0].CumulativeFrequency <= 3800) + return; + + // If we have more than 1 shift left in the model + if (--model.TimeToReorder != 0) + { + // Loop through the entries from highest to lowest, + // performing the shift on the cumulative frequencies + for (int i = model.Entries - 1; i >= 0; i--) + { + // -1, not -2; the 0 entry saves this + model.Symbols[i].CumulativeFrequency >>= 1; + if (model.Symbols[i].CumulativeFrequency <= model.Symbols[i + 1].CumulativeFrequency) + model.Symbols[i].CumulativeFrequency = (ushort)(model.Symbols[i + 1].CumulativeFrequency + 1); + } + } + + // If we have no shifts left in the model + else + { + // Reset the shifts left value to 50 + model.TimeToReorder = 50; + + // Loop through the entries setting the cumulative frequencies + for (int i = 0; i < model.Entries; i++) + { + // No -1, want to include the 0 entry + // This converts cumfreqs into frequencies, then shifts right + model.Symbols[i].CumulativeFrequency -= model.Symbols[i + 1].CumulativeFrequency; + model.Symbols[i].CumulativeFrequency++; // Avoid losing things entirely + model.Symbols[i].CumulativeFrequency >>= 1; + } + + // Now sort by frequencies, decreasing order -- this must be an + // inplace selection sort, or a sort with the same (in)stability + // characteristics + for (int i = 0; i < model.Entries - 1; i++) + { + for (int j = i + 1; j < model.Entries; j++) + { + if (model.Symbols[i].CumulativeFrequency < model.Symbols[j].CumulativeFrequency) + { + var temp = model.Symbols[i]; + model.Symbols[i] = model.Symbols[j]; + model.Symbols[j] = temp; + } + } + } + + // Then convert frequencies back to cumfreq + for (int i = model.Entries - 1; i >= 0; i--) + { + model.Symbols[i].CumulativeFrequency += model.Symbols[i + 1].CumulativeFrequency; + } + + // Then update the other part of the table + for (ushort i = 0; i < model.Entries; i++) + { + model.LookupTable[model.Symbols[i].Symbol] = i; + } + } + } + + // Bitstream reading macros (Quantum / normal byte order) + #region Macros + + /* + * These bit access routines work by using the area beyond the MSB and the + * LSB as a free source of zeroes. This avoids having to mask any bits. + * So we have to know the bit width of the bitbuffer variable. This is + * defined as Uint_BITS. + * + * Uint_BITS should be at least 16 bits. Unlike LZX's Huffman decoding, + * Quantum's arithmetic decoding only needs 1 bit at a time, it doesn't + * need an assured number. Retrieving larger bitstrings can be done with + * multiple reads and fills of the bitbuffer. The code should work fine + * for machines where Uint >= 32 bits. + * + * Also note that Quantum reads bytes in normal order; LZX is in + * little-endian order. + */ + + /// + /// Should be used first to set up the system + /// + private static void Q_INIT_BITSTREAM(out int bitsleft, out uint bitbuf) + { + bitsleft = 0; + bitbuf = 0; + } + + /// + /// Adds more data to the bit buffer, if there is room for another 16 bits. + /// + private static void Q_FILL_BUFFER(byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + if (bitsleft > 8) + return; + + byte b0 = inpos + 0 < inbuf.Length ? inbuf[inpos + 0] : (byte)0; + byte b1 = inpos + 1 < inbuf.Length ? inbuf[inpos + 1] : (byte)0; + + bitbuf |= (uint)(((b0 << 8) | b1) << (16 - bitsleft)); + bitsleft += 16; + inpos += 2; + } + + /// + /// Extracts (without removing) N bits from the bit buffer + /// + private static uint Q_PEEK_BITS(int n, uint bitbuf) + { + return bitbuf >> (32 - n); + } + + /// + /// Removes N bits from the bit buffer + /// + private static void Q_REMOVE_BITS(int n, ref int bitsleft, ref uint bitbuf) + { + bitbuf <<= n; + bitsleft -= n; + } + + /// + /// Takes N bits from the buffer and puts them in v. Unlike LZX, this can loop + /// several times to get the requisite number of bits. + /// + private static uint Q_READ_BITS(int n, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + uint v = 0; int bitrun; + for (int bitsneed = n; bitsneed != 0; bitsneed -= bitrun) + { + Q_FILL_BUFFER(inbuf, ref inpos, ref bitsleft, ref bitbuf); + bitrun = (bitsneed > bitsleft) ? bitsleft : bitsneed; + v = (v << bitrun) | Q_PEEK_BITS(bitrun, bitbuf); + Q_REMOVE_BITS(bitrun, ref bitsleft, ref bitbuf); + } + + return v; + } + + /// + /// Fetches the next symbol from the stated model and puts it in symbol. + /// It may need to read the bitstream to do this. + /// + private static ushort GET_SYMBOL(Model model, ref ushort H, ref ushort L, ref ushort C, byte[] inbuf, ref int inpos, ref int bitsleft, ref uint bitbuf) + { + ushort symf = GetFrequency(model.Symbols[0].CumulativeFrequency, H, L, C); + + int i; + for (i = 1; i < model.Entries; i++) + { + if (model.Symbols[i].CumulativeFrequency <= symf) + break; + } + + ushort symbol = model.Symbols[i - 1].Symbol; + GetCode(model.Symbols[i - 1].CumulativeFrequency, + model.Symbols[i].CumulativeFrequency, + model.Symbols[0].CumulativeFrequency, + ref H, ref L, ref C, + inbuf, ref inpos, ref bitsleft, ref bitbuf); + + UpdateModel(model, i); + return symbol; + } + + /// + /// Get the frequency for a given range and total frequency + /// + private static ushort GetFrequency(ushort totalFrequency, ushort H, ushort L, ushort C) + { + uint range = (uint)(((H - L) & 0xFFFF) + 1); + uint freq = (uint)(((C - L + 1) * totalFrequency - 1) / range); + return (ushort)(freq & 0xFFFF); + } + + /// + /// The decoder renormalization loop + /// + private static void GetCode(int previousFrequency, + int cumulativeFrequency, + int totalFrequency, + ref ushort H, + ref ushort L, + ref ushort C, + byte[] inbuf, + ref int inpos, + ref int bitsleft, + ref uint bitbuf) + { + uint range = (uint)((H - L) + 1); + H = (ushort)(L + ((previousFrequency * range) / totalFrequency) - 1); + L = (ushort)(L + (cumulativeFrequency * range) / totalFrequency); + + while (true) + { + if ((L & 0x8000) != (H & 0x8000)) + { + if ((L & 0x4000) == 0 || (H & 0x4000) != 0) + break; + + // Underflow case + C ^= 0x4000; + L &= 0x3FFF; + H |= 0x4000; + } + + L <<= 1; + H = (ushort)((H << 1) | 1); + C = (ushort)((C << 1) | Q_READ_BITS(1, inbuf, ref inpos, ref bitsleft, ref bitbuf)); + } + } + + #endregion + } +} \ No newline at end of file diff --git a/Quantum/State.cs b/Quantum/State.cs new file mode 100644 index 0000000..128c876 --- /dev/null +++ b/Quantum/State.cs @@ -0,0 +1,193 @@ +using SabreTools.Models.Compression.Quantum; + +namespace SabreTools.Compression.Quantum +{ + /// + /// + public class State + { + /// + /// The actual decoding window + /// + public byte[] Window; + + /// + /// Window size (1Kb through 2Mb) + /// + public uint WindowSize; + + /// + /// Window size when it was first allocated + /// + public uint ActualSize; + + /// + /// Current offset within the window + /// + public uint WindowPosition; + + #region Models + + /// + /// Symbol table for selector model + /// + public ModelSymbol[] SelectorModelSymbols = new ModelSymbol[7 + 1]; + + /// + /// Model for selector values + /// + public Model SelectorModel; + + /// + /// Model for Selector 0 + /// + public Model Model0; + + /// + /// Model for Selector 1 + /// + public Model Model1; + + /// + /// Model for Selector 2 + /// + public Model Model2; + + /// + /// Model for Selector 3 + /// + public Model Model3; + + /// + /// Model for Selector 4 + /// + public Model Model4; + + /// + /// Model for Selector 5 + /// + public Model Model5; + + /// + /// Model for Selector 6 Position + /// + public Model Model6Position; + + /// + /// Model for Selector 6 Length + /// + public Model Model6Length; + + #endregion + + #region Symbol Tables + + /// + /// Symbol table for Selector 0 + /// + public ModelSymbol[] Model0Symbols = new ModelSymbol[0x40 + 1]; + + /// + /// Symbol table for Selector 1 + /// + public ModelSymbol[] Model1Symbols = new ModelSymbol[0x40 + 1]; + + /// + /// Symbol table for Selector 2 + /// + public ModelSymbol[] Model2Symbols = new ModelSymbol[0x40 + 1]; + + /// + /// Symbol table for Selector 3 + /// + public ModelSymbol[] Model3Symbols = new ModelSymbol[0x40 + 1]; + + /// + /// Symbol table for Selector 4 + /// + public ModelSymbol[] Model4Symbols = new ModelSymbol[0x18 + 1]; + + /// + /// Symbol table for Selector 5 + /// + public ModelSymbol[] Model5Symbols = new ModelSymbol[0x24 + 1]; + + /// + /// Symbol table for Selector 6 Position + /// + public ModelSymbol[] Model6PositionSymbols = new ModelSymbol[0x2a + 1]; + + /// + /// Symbol table for Selector 6 Length + /// + public ModelSymbol[] Model6LengthSymbols = new ModelSymbol[0x1b + 1]; + + #endregion + + #region Decompression Tables + + /// + /// An index to the position slot bases + /// + public uint[] PositionSlotBases = new uint[42] + { + 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00006, 0x00008, 0x0000c, + 0x00010, 0x00018, 0x00020, 0x00030, 0x00040, 0x00060, 0x00080, 0x000c0, + 0x00100, 0x00180, 0x00200, 0x00300, 0x00400, 0x00600, 0x00800, 0x00c00, + 0x01000, 0x01800, 0x02000, 0x03000, 0x04000, 0x06000, 0x08000, 0x0c000, + 0x10000, 0x18000, 0x20000, 0x30000, 0x40000, 0x60000, 0x80000, 0xc0000, + 0x100000, 0x180000 + }; + + /// + /// How many bits of offset-from-base data is needed + /// + public byte[] ExtraBits = new byte[42] + { + 0, 0, 0, 0, 1, 1, 2, 2, + 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, 14, 14, + 15, 15, 16, 16, 17, 17, 18, 18, + 19, 19 + }; + + /// + /// An index to the position slot bases [Selector 6] + /// + public byte[] LengthBases = new byte[27] + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x0a, 0x0c, 0x0e, 0x12, 0x16, 0x1a, 0x1e, 0x26, + 0x2e, 0x36, 0x3e, 0x4e, 0x5e, 0x6e, 0x7e, 0x9e, + 0xbe, 0xde, 0xfe + }; + + /// + /// How many bits of offset-from-base data is needed [Selector 6] + /// + public byte[] LengthExtraBits = new byte[27] + { + 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, + 3, 3, 4, 4, 4, 4, 5, 5, + 5, 5, 0 + }; + + #endregion + + #region Decompression State + + /// + /// Bit buffer to persist between runs + /// + public uint BitBuffer = 0; + + /// + /// Bits remaining to persist between runs + /// + public int BitsLeft = 0; + + #endregion + } +} \ No newline at end of file