From 768c77a6bca88f7cc4f07a50e68823747277e71f Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 25 May 2022 21:55:36 -0700 Subject: [PATCH] Attempts at changes --- .../External/libmspack/CAB/Decompressor.cs | 78 +- BurnOutSharp/External/libmspack/CAB/Enums.cs | 1 - .../Compression/CompressionStream.ReadBits.cs | 3 +- .../Compression/CompressionStream.ReadHuff.cs | 10 +- .../libmspack/Compression/LZX.Decompress.cs | 720 +++++++++++++++--- .../External/libmspack/Compression/LZX.cs | 6 - BurnOutSharp/FileType/MicrosoftCAB.cs | 13 + 7 files changed, 675 insertions(+), 156 deletions(-) diff --git a/BurnOutSharp/External/libmspack/CAB/Decompressor.cs b/BurnOutSharp/External/libmspack/CAB/Decompressor.cs index 30044d94..8d50c233 100644 --- a/BurnOutSharp/External/libmspack/CAB/Decompressor.cs +++ b/BurnOutSharp/External/libmspack/CAB/Decompressor.cs @@ -321,49 +321,72 @@ namespace LibMSPackSharp.CAB } } - // Close existing file handles - System.Close(State?.InputFileHandle); + // Close existing output file handle in case of error System.Close(State?.OutputFileHandle); // Allocate generic decompression state - State = new DecompressState() + if (State == null) { - Folder = file.Folder, - Data = file.Folder.Data, - Offset = 0, - Block = 0, - Outlen = 0, + State = new DecompressState() + { + Folder = null, + Data = null, + System = System, + DecompressorState = null, + InputFileHandle = null, + InputCabinet = null, + }; - System = System, + State.System.Read = SysRead; + State.System.Write = SysWrite; + } - InputCabinet = file.Folder.Data.Cab, - InputFileHandle = System.Open(file.Folder.Data.Cab.Filename, OpenMode.MSPACK_SYS_OPEN_READ), - OutputFileHandle = null, // Required for skipping existing data - InputPointer = 0, - InputEnd = 0, - }; + // Do we need to change folder or reset the current folder? + if (State.Folder != file.Folder || State.Offset > file.Header.FolderOffset || State.DecompressorState == null) + { + // Do we need to open a new cab file? + if (State.InputFileHandle == null || file.Folder.Data.Cab != State.InputCabinet) + { + // Close previous file handle if from a different cab + System.Close(State?.InputFileHandle); - State.System.Read = SysRead; - State.System.Write = SysWrite; + State.InputCabinet = file.Folder.Data.Cab; + State.InputFileHandle = System.Open(file.Folder.Data.Cab.Filename, OpenMode.MSPACK_SYS_OPEN_READ); + if (State.InputFileHandle == null) + return Error = Error.MSPACK_ERR_OPEN; + } - if (State.InputFileHandle == null) - return Error = Error.MSPACK_ERR_OPEN; + // Seek to start of data blocks + // TODO: For LZX, they seem to start 8 bytes after the offset for some reason? + if (!System.Seek(State.InputFileHandle, file.Folder.Data.Offset, SeekMode.MSPACK_SYS_SEEK_START)) + return Error = Error.MSPACK_ERR_SEEK; - // Seek to start of data blocks - if (!System.Seek(State.InputFileHandle, file.Folder.Data.Offset, SeekMode.MSPACK_SYS_SEEK_START)) - return Error = Error.MSPACK_ERR_SEEK; + // Set up decompressor + if (InitDecompressionState(file.Folder.Header.CompType) != Error.MSPACK_ERR_OK) + return Error; - // Set up decompressor - if (InitDecompressionState(file.Folder.Header.CompType) != Error.MSPACK_ERR_OK) - return Error; + // Initialize new folder state + State.Folder = file.Folder; + State.Data = file.Folder.Data; + State.Offset = 0; + State.Block = 0; + State.Outlen = 0; + State.InputPointer = 0; // State.Input[0] + State.InputEnd = 0; // State.Input[0] + + // ReadError lasts for the lifetime of a decompressor + ReadError = Error.MSPACK_ERR_OK; + } - // ReadError lasts for the lifetime of a decompressor - ReadError = Error.MSPACK_ERR_OK; Error = Error.MSPACK_ERR_OK; // If file has more than 0 bytes if (filelen != 0) { + // Set the output file handle to null + State.OutputFileHandle = null; + UpdateDecompressionState(); + // Get to correct offset. // - Use null fh to say 'no writing' to cabd_sys_write() // - If SysRead() has an error, it will set self.ReadError @@ -391,6 +414,7 @@ namespace LibMSPackSharp.CAB // Close output file System.Close(State.OutputFileHandle); + State.OutputFileHandle = null; return Error; } diff --git a/BurnOutSharp/External/libmspack/CAB/Enums.cs b/BurnOutSharp/External/libmspack/CAB/Enums.cs index d0e8c55d..1279523b 100644 --- a/BurnOutSharp/External/libmspack/CAB/Enums.cs +++ b/BurnOutSharp/External/libmspack/CAB/Enums.cs @@ -109,5 +109,4 @@ namespace LibMSPackSharp.CAB /// MSCABD_PARAM_SALVAGE = 3, } - } diff --git a/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadBits.cs b/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadBits.cs index ca26417c..2313eb1e 100644 --- a/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadBits.cs +++ b/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadBits.cs @@ -11,6 +11,7 @@ */ using System; +using System.Linq; using static LibMSPackSharp.Compression.Constants; namespace LibMSPackSharp.Compression @@ -168,7 +169,7 @@ namespace LibMSPackSharp.Compression /// public void INJECT_BITS_MSB(int bitdata, int nbits) { - BitBuffer |= (uint)(bitdata << (BITBUF_WIDTH - nbits - BitsLeft)); + BitBuffer |= ((uint)bitdata << (BITBUF_WIDTH - nbits - BitsLeft)); BitsLeft += nbits; } diff --git a/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadHuff.cs b/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadHuff.cs index e8e73b6d..f2165cbb 100644 --- a/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadHuff.cs +++ b/BurnOutSharp/External/libmspack/Compression/CompressionStream.ReadHuff.cs @@ -77,11 +77,11 @@ namespace LibMSPackSharp.Compression public static bool MakeDecodeTableMSB(int nsyms, int nbits, byte[] length, ushort[] table) { ushort sym, next_symbol; - uint leaf, fill; + long leaf, fill; byte bit_num; - uint pos = 0; // The current position in the decode table - uint table_mask = (uint)1 << nbits; - uint bit_mask = table_mask >> 1; // Don't do 0 length codes + long pos = 0; // The current position in the decode table + long table_mask = 1 << nbits; + long bit_mask = table_mask >> 1; // Don't do 0 length codes // Fill entries for codes short enough for a direct mapping for (bit_num = 1; bit_num <= nbits; bit_num++) @@ -116,7 +116,7 @@ namespace LibMSPackSharp.Compression } // next_symbol = base of allocation for long codes - next_symbol = ((table_mask >> 1) < nsyms) ? (ushort)nsyms : (ushort)(table_mask >> 1); + next_symbol = (ushort)(((table_mask >> 1) < nsyms) ? nsyms : (table_mask >> 1)); // Give ourselves room for codes to grow by up to 16 more bits. // codes now start at bit nbits+16 and end at (nbits+16-codelength) diff --git a/BurnOutSharp/External/libmspack/Compression/LZX.Decompress.cs b/BurnOutSharp/External/libmspack/Compression/LZX.Decompress.cs index fc6ad0c3..48a94c00 100644 --- a/BurnOutSharp/External/libmspack/Compression/LZX.Decompress.cs +++ b/BurnOutSharp/External/libmspack/Compression/LZX.Decompress.cs @@ -76,6 +76,7 @@ using System; using System.IO; +using System.Linq; using static LibMSPackSharp.Compression.Constants; namespace LibMSPackSharp.Compression @@ -189,7 +190,6 @@ namespace LibMSPackSharp.Compression OutputPointer = 0, OutputEnd = 0, - OutputIsE8 = true, }; lzx.ResetState(); @@ -282,13 +282,530 @@ namespace LibMSPackSharp.Compression /// should be considered unusable and lzxd_decompress() should not be /// called again on this stream. /// - /// LZX decompression state, as allocated by lzxd_init(). /// the number of bytes of data to decompress. /// an error code, or MSPACK_ERR_OK if successful public Error Decompress(long out_bytes) { - int warned = 0; + int match_length, length_footer, extra, verbatim_bits, bytes_todo; + int this_run, main_element, aligned_bits, j, warned = 0; byte[] buf = new byte[12]; + int runsrc, rundest; + int frame_size = 0, end_frame, match_offset; + + // Easy answers + if (out_bytes < 0) + return Error.MSPACK_ERR_ARGS; + if (Error != Error.MSPACK_ERR_OK) + return Error; + + // Flush out any stored-up bytes before we begin + int i = OutputEnd - OutputPointer; + if (i > out_bytes) + i = (int)out_bytes; + + if (i != 0) + { + if (System.Write(OutputFileHandle, IntelStarted ? E8Buffer : Window, OutputPointer, i) != i) + return Error = Error.MSPACK_ERR_WRITE; + + OutputPointer += i; + Offset += i; + out_bytes -= i; + } + + if (out_bytes == 0) + return Error.MSPACK_ERR_OK; + + end_frame = (int)((uint)((Offset + out_bytes) / LZX_FRAME_SIZE) + 1); + + while (Frame < end_frame) + { + // Have we reached the reset interval? (if there is one?) + if (ResetInterval != 0 && ((Frame & ResetInterval) == 0)) + { + if (BlockRemaining != 0) + { + // This is a file format error, we can make a best effort to extract what we can + Console.WriteLine($"{BlockRemaining} bytes remaining at reset interval"); + if (warned == 0) + { + System.Message(null, "WARNING; invalid reset interval detected during LZX decompression"); + warned++; + } + } + + // Re-read the intel header and reset the huffman lengths + ResetState(); + } + + // LZX DELTA format has chunk_size, not present in LZX format + if (IsDelta) + { + ENSURE_BITS(16); + REMOVE_BITS_MSB(16); + } + + // Read header if necessary + if (HeaderRead == 0) + { + // Read 1 bit. If bit=0, intel filesize = 0. + // If bit=1, read intel filesize (32 bits) + j = 0; + i = (int)READ_BITS_MSB(1); + if (i != 0) + { + i = (int)READ_BITS_MSB(16); + j = (int)READ_BITS_MSB(16); + } + + IntelFileSize = (i << 16) | j; + HeaderRead = 1; + } + + // Calculate size of frame: all frames are 32k except the final frame + // which is 32kb or less. this can only be calculated when lzx->length + // has been filled in. + frame_size = LZX_FRAME_SIZE; + if (Length != 0 && (Length - Offset) < frame_size) + frame_size = (int)(Length - Offset); + + // Decode until one more frame is available + bytes_todo = (int)(FramePosition + frame_size - WindowPosition); + while (bytes_todo > 0) + { + // Initialise new block, if one is needed + if (BlockRemaining == 0) + { + // Realign if previous block was an odd-sized UNCOMPRESSED block + if (BlockType == LZXBlockType.LZX_BLOCKTYPE_UNCOMPRESSED && (BlockLength & 1) != 0) + { + READ_IF_NEEDED(); + InputPointer++; + } + + // Read block type (3 bits) and block length (24 bits) + BlockType = (LZXBlockType)READ_BITS_MSB(3); + i = (int)READ_BITS_MSB(16); + j = (int)READ_BITS_MSB(8); + BlockRemaining = BlockLength = (i << 8) | j; + Console.WriteLine($"new block t{BlockType} len {BlockLength}"); + + // Read individual block headers + switch (BlockType) + { + case LZXBlockType.LZX_BLOCKTYPE_ALIGNED: + // Read lengths of and build aligned huffman decoding tree + for (i = 0; i < 8; i++) + { + j = (int)READ_BITS_MSB(3); + ALIGNED_len[i] = (byte)j; + } + + // Rest of aligned header is same as verbatim + + // Read lengths of and build main huffman decoding tree + READ_LENGTHS(MAINTREE_len, 0, 256); + READ_LENGTHS(MAINTREE_len, 256, LZX_NUM_CHARS + NumOffsets); + BUILD_TABLE(MAINTREE_table, MAINTREE_len, LZX_MAINTREE_TABLEBITS, LZX_MAINTREE_MAXSYMBOLS); + + // If the literal 0xE8 is anywhere in the block... + if (MAINTREE_len[0xE8] != 0) + IntelStarted = true; + + // Read lengths of and build lengths huffman decoding tree + READ_LENGTHS(LENGTH_len, 0, LZX_NUM_SECONDARY_LENGTHS); + BUILD_TABLE_MAYBE_EMPTY(); + break; + + case LZXBlockType.LZX_BLOCKTYPE_VERBATIM: + // Read lengths of and build main huffman decoding tree + READ_LENGTHS(MAINTREE_len, 0, 256); + READ_LENGTHS(MAINTREE_len, 256, LZX_NUM_CHARS + NumOffsets); + BUILD_TABLE(MAINTREE_table, MAINTREE_len, LZX_MAINTREE_TABLEBITS, LZX_MAINTREE_MAXSYMBOLS); + + // If the literal 0xE8 is anywhere in the block... + if (MAINTREE_len[0xE8] != 0) + IntelStarted = true; + + // Read lengths of and build lengths huffman decoding tree + READ_LENGTHS(LENGTH_len, 0, LZX_NUM_SECONDARY_LENGTHS); + BUILD_TABLE_MAYBE_EMPTY(); + break; + + case LZXBlockType.LZX_BLOCKTYPE_UNCOMPRESSED: + // Because we can't assume otherwise + IntelStarted = true; + + // Read 1-16 (not 0-15) bits to align to bytes + if (BitsLeft == 0) + ENSURE_BITS(16); + + BitsLeft = 0; + BitBuffer = 0; + + // Read 12 bytes of stored R0 / R1 / R2 values + for (rundest = 0, i = 0; i < 12; i++) + { + READ_IF_NEEDED(); + buf[rundest++] = InputBuffer[InputPointer++]; + } + + R[0] = (uint)(buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24)); + R[1] = (uint)(buf[4] | (buf[5] << 8) | (buf[6] << 16) | (buf[7] << 24)); + R[2] = (uint)(buf[8] | (buf[9] << 8) | (buf[10] << 16) | (buf[11] << 24)); + break; + + default: + Console.WriteLine($"Bad block type {BlockType}"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + } + + // Decode more of the block: + // run = min(what's available, what's needed) + this_run = BlockRemaining; + if (this_run > bytes_todo) + this_run = bytes_todo; + + // Assume we decode exactly this_run bytes, for now + bytes_todo -= this_run; + BlockRemaining -= this_run; + + // Decode at least this_run bytes + switch (BlockType) + { + case LZXBlockType.LZX_BLOCKTYPE_ALIGNED: + case LZXBlockType.LZX_BLOCKTYPE_VERBATIM: + while (this_run > 0) + { + main_element = (int)READ_HUFFSYM_MSB(MAINTREE_table, MAINTREE_len, LZX_MAINTREE_TABLEBITS, LZX_MAINTREE_MAXSYMBOLS); + if (main_element < LZX_NUM_CHARS) + { + // Literal: 0 to LZX_NUM_CHARS-1 + Window[WindowPosition++] = (byte)main_element; + this_run--; + } + else + { + // Match: LZX_NUM_CHARS ((slot<<3) | length_header (3 bits)) + main_element -= LZX_NUM_CHARS; + + // Get match length + match_length = main_element & LZX_NUM_PRIMARY_LENGTHS; + if (match_length == LZX_NUM_PRIMARY_LENGTHS) + { + if (LENGTH_empty != 0) + { + Console.WriteLine("LENGTH symbol needed but tree is empty"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + length_footer = (int)READ_HUFFSYM_MSB(LENGTH_table, LENGTH_len, LZX_LENGTH_TABLEBITS, LZX_LENGTH_MAXSYMBOLS); + match_length += length_footer; + } + + match_length += LZX_MIN_MATCH; + + // Get match offset + switch ((match_offset = (main_element >> 3))) + { + case 0: + match_offset = (int)R[0]; + break; + + case 1: + match_offset = (int)R[1]; + R[1] = R[0]; + R[0] = (uint)match_offset; + break; + + case 2: + match_offset = (int)R[2]; + R[2] = R[0]; + R[0] = (uint)match_offset; + break; + + default: + if (BlockType == LZXBlockType.LZX_BLOCKTYPE_VERBATIM) + { + if (match_offset == 3) + { + match_offset = 1; + } + else + { + extra = (match_offset >= 36) ? 17 : LZXExtraBits[match_offset]; + verbatim_bits = (int)READ_BITS_MSB(extra); + match_offset = (int)(LZXPositionBase[match_offset] - 2 + verbatim_bits); + } + } + else // LZX_BLOCKTYPE_ALIGNED + { + extra = (match_offset >= 36) ? 17 : LZXExtraBits[match_offset]; + match_offset = (int)(LZXPositionBase[match_offset] - 2); + if (extra > 3) // >3: verbatim and aligned bits + { + extra -= 3; + verbatim_bits = (int)READ_BITS_MSB(extra); + match_offset += (verbatim_bits << 3); + aligned_bits = (int)READ_HUFFSYM_MSB(ALIGNED_table, ALIGNED_len, LZX_ALIGNED_TABLEBITS, LZX_ALIGNED_MAXSYMBOLS); + match_offset += aligned_bits; + } + else if (extra == 3) // 3: aligned bits only + { + aligned_bits = (int)READ_HUFFSYM_MSB(ALIGNED_table, ALIGNED_len, LZX_ALIGNED_TABLEBITS, LZX_ALIGNED_MAXSYMBOLS); + match_offset += aligned_bits; + } + else if (extra > 0) // 1-2: verbatim bits only + { + verbatim_bits = (int)READ_BITS_MSB(extra); + match_offset += verbatim_bits; + } + else // 0: not defined in LZX specification! + { + match_offset = 1; + } + } + + // Update repeated offset LRU queue + R[2] = R[1]; + R[1] = R[0]; + R[0] = (uint)match_offset; + break; + } + + // LZX DELTA uses max match length to signal even longer match + if (match_length == LZX_MAX_MATCH && IsDelta) + { + int extra_len = 0; + ENSURE_BITS(3); // 4 entry huffman tree + if (PEEK_BITS_MSB(1) == 0) + { + REMOVE_BITS_MSB(1); // '0' -> 8 extra length bits + extra_len = (int)READ_BITS_MSB(8); + } + else if (PEEK_BITS_MSB(2) == 2) + { + REMOVE_BITS_MSB(2); // '10' -> 10 extra length bits + 0x100 + extra_len = (int)READ_BITS_MSB(10); + extra_len += 0x100; + } + else if (PEEK_BITS_MSB(3) == 6) + { + REMOVE_BITS_MSB(3); // '110' -> 12 extra length bits + 0x500 + extra_len = (int)READ_BITS_MSB(12); + extra_len += 0x500; + } + else + { + REMOVE_BITS_MSB(3); // '111' -> 15 extra length bits + extra_len = (int)READ_BITS_MSB(15); + } + + match_length += extra_len; + } + + if ((WindowPosition + match_length) > WindowSize) + { + Console.WriteLine("Match ran over window wrap"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + // Copy match + rundest = WindowPosition; + i = match_length; + + // Does match offset wrap the window? + if (match_offset > WindowPosition) + { + if (match_offset > Offset && (match_offset - WindowPosition) > ReferenceDataSize) + { + Console.WriteLine("Match offset beyond LZX stream"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + // j = length from match offset to end of window + j = match_offset - WindowPosition; + if (j > (int)WindowSize) + { + Console.WriteLine("Match offset beyond window boundaries"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + runsrc = (int)(WindowSize - j); + if (j < i) + { + // If match goes over the window edge, do two copy runs + i -= j; + while (j-- > 0) + { + Window[rundest++] = Window[runsrc++]; + } + + runsrc = 0; + } + + while (i-- > 0) + { + Window[rundest++] = Window[runsrc++]; + } + } + else + { + runsrc = rundest - match_offset; + while (i-- > 0) + { + Window[rundest++] = Window[runsrc++]; + } + } + + this_run -= match_length; + WindowPosition += match_length; + } + } + + break; + + case LZXBlockType.LZX_BLOCKTYPE_UNCOMPRESSED: + // As this_run is limited not to wrap a frame, this also means it + // won't wrap the window (as the window is a multiple of 32k) + rundest = WindowPosition; + WindowPosition += this_run; + while (this_run > 0) + { + if ((i = InputEnd - InputPointer) == 0) + { + READ_IF_NEEDED(); + } + else + { + if (i > this_run) + i = this_run; + + Array.Copy(InputBuffer, InputPointer, Window, rundest, i); + rundest += 1; + InputPointer += i; + this_run -= i; + } + } + + break; + + default: + return Error = Error.MSPACK_ERR_DECRUNCH; // Might as well + } + + // Did the final match overrun our desired this_run length? + if (this_run < 0) + { + if ((uint)(-this_run) > BlockRemaining) + { + Console.Write($"Overrun went past end of block by {-this_run} ({BlockRemaining} remaining)"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + BlockRemaining -= -this_run; + } + } + + // Streams don't extend over frame boundaries + if ((WindowPosition - FramePosition) != frame_size) + { + Console.WriteLine($"Decode beyond output frame limits {WindowPosition - FramePosition} != {frame_size}"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + // Re-align input bitstream + if (BitsLeft > 0) + ENSURE_BITS(16); + if ((BitsLeft & 15) != 0) + REMOVE_BITS_MSB(BitsLeft & 15); + + // Check that we've used all of the previous frame first + if (OutputPointer != OutputEnd) + { + Console.Write($"{OutputEnd - OutputPointer} avail bytes, new {frame_size} frame"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + // Does this intel block _really_ need decoding? + if (IntelStarted && IntelFileSize != 0 && Frame < 32768 && frame_size > 10) + { + int data = 0; + int dataend = frame_size - 10; + int curpos = (int)Offset; + int filesize = IntelFileSize; + int abs_off, rel_off; + + // Copy E8 block to the e8 buffer and tweak if needed + OutputPointer = 0; + Array.Copy(Window, FramePosition, E8Buffer, data, frame_size); + + while (data < dataend) + { + if (E8Buffer[data++] != 0xE8) + { + curpos++; + continue; + } + + abs_off = E8Buffer[data + 0] | (E8Buffer[data + 1] << 8) | (E8Buffer[data + 2] << 16) | (E8Buffer[data + 3] << 24); + if ((abs_off >= -curpos) && (abs_off < filesize)) + { + rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize; + E8Buffer[data + 0] = (byte)rel_off; + E8Buffer[data + 1] = (byte)(rel_off >> 8); + E8Buffer[data + 2] = (byte)(rel_off >> 16); + E8Buffer[data + 3] = (byte)(rel_off >> 24); + } + + data += 4; + curpos += 5; + } + } + else + { + IntelStarted = false; + OutputPointer = (int)FramePosition; + } + + OutputEnd = frame_size; + + // Write a frame + i = (int)(out_bytes < frame_size ? out_bytes : frame_size); + if (System.Write(OutputFileHandle, IntelStarted ? E8Buffer : Window, OutputPointer, i) != i) + return Error = Error.MSPACK_ERR_WRITE; + + OutputPointer += i; + Offset += i; + out_bytes -= i; + + // Advance frame start position + FramePosition += (uint)frame_size; + Frame++; + + // Wrap window / frame position pointers + if (WindowPosition == WindowSize) + WindowPosition = 0; + if (FramePosition == WindowSize) + FramePosition = 0; + } + + if (out_bytes != 0) + { + Console.WriteLine($"{out_bytes} bytes left to output"); + return Error = Error.MSPACK_ERR_DECRUNCH; + } + + return Error.MSPACK_ERR_OK; + } + + /// + /// Use the cleaned up code for decompression based on wimlib + /// + public Error DecompressNew(long out_bytes) + { + int warned = 0; // Easy answers if (out_bytes < 0) @@ -304,7 +821,7 @@ namespace LibMSPackSharp.Compression if (leftover_bytes != 0) { - try { System.Write(OutputFileHandle, OutputIsE8 ? E8Buffer : Window, OutputPointer, leftover_bytes); } + try { System.Write(OutputFileHandle, Window, OutputPointer, leftover_bytes); } catch { return Error = Error.MSPACK_ERR_WRITE; } OutputPointer += leftover_bytes; @@ -355,7 +872,7 @@ namespace LibMSPackSharp.Compression int bytes_todo = (int)(FramePosition + frame_size - WindowPosition); while (bytes_todo > 0) { - ReadBlockHeader(buf); + ReadBlockHeader(); if (Error != Error.MSPACK_ERR_OK) return Error; @@ -397,9 +914,7 @@ namespace LibMSPackSharp.Compression } else { - if (i > this_run) - i = this_run; - + i = Math.Min(i, this_run); Array.Copy(InputBuffer, InputPointer, Window, rundest, i); rundest += i; @@ -409,7 +924,7 @@ namespace LibMSPackSharp.Compression } // Realign if this was an odd-sized UNCOMPRESSED block - if (InputFileHandle.Position != InputFileHandle.Length - 1 && (BlockLength & 1) != 0) + if (InputPointer != InputEnd - 1 && (BlockLength & 1) != 0) { READ_IF_NEEDED(); if (Error != Error.MSPACK_ERR_OK) @@ -462,20 +977,14 @@ namespace LibMSPackSharp.Compression // Does this intel block _really_ need decoding? if (IntelStarted) - { - UndoE8Preprocessing(frame_size); - } - else - { - OutputIsE8 = false; - OutputPointer = (int)FramePosition; - } + UndoE8Preprocessing((int)FramePosition, out_bytes); + OutputPointer = (int)FramePosition; OutputEnd = (int)(OutputPointer + frame_size); // Write a frame int new_out_bytes = (int)((out_bytes < frame_size) ? out_bytes : frame_size); - try { System.Write(OutputFileHandle, OutputIsE8 ? E8Buffer : Window, OutputPointer, new_out_bytes); } + try { System.Write(OutputFileHandle, Window, OutputPointer, new_out_bytes); } catch { return Error = Error.MSPACK_ERR_WRITE; } OutputPointer += new_out_bytes; @@ -537,9 +1046,19 @@ namespace LibMSPackSharp.Compression private Error READ_LENGTHS(byte[] lengths, uint first, uint last) { + int i_ptr = InputPointer; + int i_end = InputEnd; + uint bit_buffer = BitBuffer; + int bits_left = BitsLeft; + if (ReadLens(lengths, first, last) != Error.MSPACK_ERR_OK) return Error; + InputPointer = i_ptr; + InputEnd = i_end; + BitBuffer = bit_buffer; + BitsLeft = bits_left; + return Error = Error.MSPACK_ERR_OK; } @@ -593,7 +1112,7 @@ namespace LibMSPackSharp.Compression main_element -= LZX_NUM_CHARS; // The length header consists of the lower 3 bits of the main element. - // The position slot is the rest of it. + // The position slot is the rest of it. int length_header = main_element & LZX_NUM_PRIMARY_LENGTHS; int position_slot = main_element >> 3; @@ -747,7 +1266,7 @@ namespace LibMSPackSharp.Compression return Error = Error.MSPACK_ERR_OK; } - private Error ReadBlockHeader(byte[] buffer) + private Error ReadBlockHeader() { ENSURE_BITS(4); @@ -763,20 +1282,20 @@ namespace LibMSPackSharp.Compression } else { - int tmp; + uint tmp; block_size = 0; - tmp = (int)READ_BITS_MSB(8); - block_size |= tmp; - tmp = (int)READ_BITS_MSB(8); + tmp = (uint)READ_BITS_MSB(8); + block_size |= (int)tmp; + tmp = (uint)READ_BITS_MSB(8); block_size <<= 8; - block_size |= tmp; + block_size |= (int)tmp; if (WindowSize >= 65536) { - tmp = (int)READ_BITS_MSB(8); + tmp = (uint)READ_BITS_MSB(8); block_size <<= 8; - block_size |= tmp; + block_size |= (int)tmp; } } @@ -851,26 +1370,24 @@ namespace LibMSPackSharp.Compression break; case LZXBlockType.LZX_BLOCKTYPE_UNCOMPRESSED: - // Read 1-16 (not 0-15) bits to align to bytes if (BitsLeft == 0) - ENSURE_BITS(16); - - BitsLeft = 0; BitBuffer = 0; - - // Read 12 bytes of stored R[0] / R[1] / R[2] values - for (int rundest = 0, k = 0; k < 12; k++) { - READ_IF_NEEDED(); - if (Error != Error.MSPACK_ERR_OK) - return Error; - - buffer[rundest++] = InputBuffer[InputPointer++]; + ENSURE_BITS(16); + BlockRemaining -= 2; + } + else + { + BitsLeft = 0; + BitBuffer = 0; } // TODO: uint[] R should be a part of a state object - R[0] = (uint)(buffer[0] | (buffer[1] << 8) | (buffer[2] << 16) | (buffer[3] << 24)); - R[1] = (uint)(buffer[4] | (buffer[5] << 8) | (buffer[6] << 16) | (buffer[7] << 24)); - R[2] = (uint)(buffer[8] | (buffer[9] << 8) | (buffer[10] << 16) | (buffer[11] << 24)); + R[0] = BitConverter.ToUInt32(new Span(InputBuffer, InputPointer + 0, 4).ToArray().Reverse().ToArray(), 0); + R[1] = BitConverter.ToUInt32(new Span(InputBuffer, InputPointer + 4, 4).ToArray().Reverse().ToArray(), 0); + R[2] = BitConverter.ToUInt32(new Span(InputBuffer, InputPointer + 8, 4).ToArray().Reverse().ToArray(), 0); + + InputPointer += 12; + BlockRemaining -= 12; break; @@ -884,69 +1401,71 @@ namespace LibMSPackSharp.Compression private Error ReadLens(byte[] lens, uint first, uint last) { + uint x, y; + int z; + // Read lengths for pretree (20 symbols, lengths stored in fixed 4 bits) - for (int i = 0; i < LZX_PRETREE_MAXSYMBOLS; i++) + for (x = 0; x < LZX_PRETREE_MAXSYMBOLS; x++) { - uint y = (uint)READ_BITS_MSB(4); - PRETREE_len[i] = (byte)y; + y = (uint)READ_BITS_MSB(4); + PRETREE_len[x] = (byte)y; } BUILD_TABLE(PRETREE_table, PRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS); if (Error != Error.MSPACK_ERR_OK) return Error; - for (uint lensPtr = first; lensPtr < last;) + for (x = first; x < last;) { - uint num_zeroes, num_same; - int tree_code = (int)READ_HUFFSYM_MSB(PRETREE_table, PRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS); - switch (tree_code) + z = (int)READ_HUFFSYM_MSB(PRETREE_table, PRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS); + switch (z) { // Code = 17, run of ([read 4 bits]+4) zeros case 17: - num_zeroes = (uint)READ_BITS_MSB(4); - num_zeroes += 4; - while (num_zeroes-- != 0) + y = (uint)READ_BITS_MSB(4); + y += 4; + while (y-- != 0) { - lens[lensPtr++] = 0; + lens[x++] = 0; } break; // Code = 18, run of ([read 5 bits]+20) zeros case 18: - num_zeroes = (uint)READ_BITS_MSB(5); - num_zeroes += 20; - while (num_zeroes-- != 0) + y = (uint)READ_BITS_MSB(5); + y += 20; + while (y-- != 0) { - lens[lensPtr++] = 0; + lens[x++] = 0; } break; // Code = 19, run of ([read 1 bit]+4) [read huffman symbol] case 19: - num_same = (uint)READ_BITS_MSB(1); - num_same += 4; + y = (uint)READ_BITS_MSB(1); + y += 4; - tree_code = (int)READ_HUFFSYM_MSB(PRETREE_table, PRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS); - tree_code = lens[lensPtr] - tree_code; - if (tree_code < 0) - tree_code += 17; + z = (int)READ_HUFFSYM_MSB(PRETREE_table, PRETREE_len, LZX_PRETREE_TABLEBITS, LZX_PRETREE_MAXSYMBOLS); + z = lens[x] - z; + if (z < 0) + z += 17; - while (num_same-- != 0) + while (y-- != 0) { - lens[lensPtr++] = (byte)tree_code; + lens[x++] = (byte)z; } break; // Code = 0 to 16, delta current length entry default: - tree_code = lens[lensPtr] - tree_code; - if (tree_code < 0) - tree_code += 17; + z = lens[x] - z; + if (z < 0) + z += 17; - lens[lensPtr++] = (byte)tree_code; + lens[x++] = (byte)z; break; } } @@ -975,81 +1494,50 @@ namespace LibMSPackSharp.Compression } } - private void UndoE8Preprocessing(uint frame_size) + private void UndoE8Preprocessing(int data, long out_bytes) { - if (frame_size > 10) + int p8 = data; + if (out_bytes > 10) { // Finish any bytes that weren't processed by the vectorized implementation. - int start = WindowPosition; - int p8_end = (int)(WindowPosition + frame_size - 10); + int p8_end = (int)(out_bytes - 10); do { - if (Window[WindowPosition] == 0xe8) + if (Window[p8] == 0xe8) { + int target = p8 + 1; + int input_pos = p8 - data; + int abs_offset, rel_offset; // XXX: This assumes unaligned memory accesses are okay. - abs_offset = BitConverter.ToInt32(Window, WindowPosition + 1); + abs_offset = BitConverter.ToInt32(new Span(Window, target, 4).ToArray().Reverse().ToArray(), 0); if (abs_offset >= 0) { if (abs_offset < 12_000_000) { // "good translation" - rel_offset = abs_offset - start; - Array.Copy(BitConverter.GetBytes(rel_offset), 0, Window, WindowPosition + 1, 4); + rel_offset = abs_offset - input_pos; + Array.Copy(BitConverter.GetBytes(rel_offset).Reverse().ToArray(), 0, Window, target, 4); } } else { - if (abs_offset >= -start) + if (abs_offset >= -input_pos) { // "compensating translation" rel_offset = abs_offset + 12_000_000; - Array.Copy(BitConverter.GetBytes(rel_offset), 0, Window, WindowPosition + 1, 4); + Array.Copy(BitConverter.GetBytes(rel_offset).Reverse().ToArray(), 0, Window, target, 4); } } - WindowPosition += 5; + p8 += 5; } else { - WindowPosition++; + p8++; } - } while (WindowPosition < p8_end); - } - - - int data = 0; - int dataend = (int)(frame_size - 10); - int curpos = (int)Offset; - int filesize = IntelFileSize; - int abs_off, rel_off; - - // Copy e8 block to the e8 buffer and tweak if needed - OutputIsE8 = true; - OutputPointer = data; - Array.Copy(Window, FramePosition, E8Buffer, data, frame_size); - - while (data < dataend) - { - if (E8Buffer[data++] != 0xE8) - { - curpos++; - continue; - } - - abs_off = E8Buffer[data + 0] | (E8Buffer[data + 1] << 8) | (E8Buffer[data + 2] << 16) | (E8Buffer[data + 3] << 24); - if ((abs_off >= -curpos) && (abs_off < filesize)) - { - rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize; - E8Buffer[data + 0] = (byte)rel_off; - E8Buffer[data + 1] = (byte)(rel_off >> 8); - E8Buffer[data + 2] = (byte)(rel_off >> 16); - E8Buffer[data + 3] = (byte)(rel_off >> 24); - } - - data += 4; - curpos += 5; + } while (p8 < p8_end); } } } diff --git a/BurnOutSharp/External/libmspack/Compression/LZX.cs b/BurnOutSharp/External/libmspack/Compression/LZX.cs index acc76f04..810adc03 100644 --- a/BurnOutSharp/External/libmspack/Compression/LZX.cs +++ b/BurnOutSharp/External/libmspack/Compression/LZX.cs @@ -126,12 +126,6 @@ namespace LibMSPackSharp.Compression public byte LENGTH_empty { get; set; } - // This is used purely for doing the intel E8 transform public byte[] E8Buffer { get; set; } = new byte[LZX_FRAME_SIZE]; - - /// - /// Is the output pointer referring to E8? - /// - public bool OutputIsE8 { get; set; } } } diff --git a/BurnOutSharp/FileType/MicrosoftCAB.cs b/BurnOutSharp/FileType/MicrosoftCAB.cs index bdb76d74..b2b70a1f 100644 --- a/BurnOutSharp/FileType/MicrosoftCAB.cs +++ b/BurnOutSharp/FileType/MicrosoftCAB.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.IO; +using System.Text.RegularExpressions; using BurnOutSharp.Interfaces; using BurnOutSharp.Tools; using LibMSPackSharp; @@ -52,6 +53,18 @@ namespace BurnOutSharp.FileType return null; } + // If there are additional previous CABs, add those + while (!string.IsNullOrWhiteSpace(cabFile.NextName)) + { + // TODO: Implement + } + + // If there are additional next CABs, add those + while (!string.IsNullOrWhiteSpace(cabFile.NextName)) + { + // TODO: Implement + } + // Loop through the found internal files var sub = cabFile.Files; while (sub != null)