diff --git a/README.MD b/README.MD index cd2dded..3ef5f3d 100644 --- a/README.MD +++ b/README.MD @@ -28,5 +28,5 @@ For the latest WIP build here: [Rolling Release](https://github.com/SabreTools/S | Library Name | Use | | --- | ---| -| [DotNetZip](https://github.com/DinoChiesa/DotNetZip) | DEFLATE implementation; minor edits have been made | +| [DotNetZip](https://github.com/DinoChiesa/DotNetZip) | BZip2 and DEFLATE implementations; minor edits have been made | | [ZLibPort](https://github.com/Nanook/zlib-C-To-CSharp-Port) | Adds zlib code for internal and external use; minor edits have been made | diff --git a/SabreTools.Compression/BZip2/BZip2.cs b/SabreTools.Compression/BZip2/BZip2.cs new file mode 100644 index 0000000..0c0013a --- /dev/null +++ b/SabreTools.Compression/BZip2/BZip2.cs @@ -0,0 +1,114 @@ +// BZip2InputStream.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-July-31 11:57:32> +// +// ------------------------------------------------------------------ +// +// This module defines the BZip2InputStream class, which is a decompressing +// stream that handles BZIP2. This code is derived from Apache commons source code. +// The license below applies to the original Apache code. +// +// ------------------------------------------------------------------ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * to whom the Ant project is very grateful for his + * great code. + */ + +// compile: msbuild +// not: csc.exe /t:library /debug+ /out:SabreTools.Compression.BZip2.dll BZip2InputStream.cs BCRC32.cs Rand.cs + +namespace SabreTools.Compression.BZip2 +{ + // /** + // * Checks if the signature matches what is expected for a bzip2 file. + // * + // * @param signature + // * the bytes to check + // * @param length + // * the number of bytes to check + // * @return true, if this stream is a bzip2 compressed stream, false otherwise + // * + // * @since Apache Commons Compress 1.1 + // */ + // public static boolean MatchesSig(byte[] signature) + // { + // if ((signature.Length < 3) || + // (signature[0] != 'B') || + // (signature[1] != 'Z') || + // (signature[2] != 'h')) + // return false; + // + // return true; + // } + + internal static class BZip2 + { + internal static T[][] InitRectangularArray(int d1, int d2) + { + var x = new T[d1][]; + for (int i = 0; i < d1; i++) + { + x[i] = new T[d2]; + } + return x; + } + + public static readonly int BlockSizeMultiple = 100000; + public static readonly int MinBlockSize = 1; + public static readonly int MaxBlockSize = 9; + public static readonly int MaxAlphaSize = 258; + public static readonly int MaxCodeLength = 23; + public static readonly char RUNA = (char)0; + public static readonly char RUNB = (char)1; + public static readonly int NGroups = 6; + public static readonly int G_SIZE = 50; + public static readonly int N_ITERS = 4; + public static readonly int MaxSelectors = (2 + (900000 / G_SIZE)); + public static readonly int NUM_OVERSHOOT_BYTES = 20; + /* + *

If you are ever unlucky/improbable enough to get a stack + * overflow whilst sorting, increase the following constant and + * try again. In practice I have never seen the stack go above 27 + * elems, so the following limit seems very generous.

+ */ + internal static readonly int QSORT_STACK_SIZE = 1000; + + + } + +} \ No newline at end of file diff --git a/SabreTools.Compression/BZip2/BZip2Compressor.cs b/SabreTools.Compression/BZip2/BZip2Compressor.cs new file mode 100644 index 0000000..43c6131 --- /dev/null +++ b/SabreTools.Compression/BZip2/BZip2Compressor.cs @@ -0,0 +1,1942 @@ +// BZip2Compressor.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-July-28 06:17:22> +// +// ------------------------------------------------------------------ +// +// This module defines the BZip2Compressor class, which is a +// BZIP2-compressing encoder. It is used internally in the BZIP2 +// library, by the BZip2OutputStream class and its parallel variant, +// ParallelBZip2OutputStream. This code was originally based on Apache +// commons source code, and significantly modified. The license below +// applies to the original Apache code and to this modified variant. +// +// ------------------------------------------------------------------ + + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * to whom the Ant project is very grateful for his + * great code. + */ + + +// +// Design notes: +// +// This class performs BZip2 compression. It is derived from the +// BZip2OutputStream from the Apache commons source code, but is +// significantly modified from that code. While the Apache class is a +// stream that compresses, this particular class simply performs +// compression. It follows a Manager pattern. It manages an internal +// buffer for uncompressed data; callers place data into the buffer +// using the Fill() method. This class then compresses the data and +// writes the compressed form out, via the CompressAndWrite() method. +// Because BZip2 uses byte-shredding, this class relies on a BitWriter, +// and not a .NET Stream, to emit its output. (*Think of the BitWriter +// class as an Adapter that enables Bit-oriented output to a standard +// byte-oriented .NET stream.) +// +// This class exists to support the two distinct output streams that +// perform BZip2 compression: BZip2OutputStream and +// ParallelBZip2OutputStream. These streams rely on BZip2Compressor to +// provide the encoder/compression logic. This code has been derived +// from the bzip2 output stream in the Apache commons library; it has +// been significantly modified from that form, in order to provide a +// single compressor that could support both types of streams. +// +// In a bz2 file or stream, there is never any bit padding except for 0..7 +// bits in the final byte in the file. Successive compressed blocks in a +// .bz2 file are not byte-aligned. +// +// + +using System; +using System.IO; + +// flymake: csc.exe /t:module BZip2InputStream.cs BZip2OutputStream.cs Rand.cs BCRC32.cs @@FILE@@ + +namespace SabreTools.Compression.BZip2 +{ + internal class BZip2Compressor + { + private int blockSize100k; // 0...9 + private int currentByte = -1; + private int runLength = 0; + private int last; // index into the block of the last char processed + private int outBlockFillThreshold; + private CompressionState cstate; + private readonly CRC32 crc = new CRC32(true); + BitWriter bw; + int runs; + + /* + * The following three vars are used when sorting. If too many long + * comparisons happen, we stop sorting, randomise the block slightly, and + * try again. I think this wrinkle in the implementation was removed from + * a later rev of the C-language bzip, not sure. -DPC 24 Jul 2011 + * + */ + private int workDone; + private int workLimit; + private bool firstAttempt; + private bool blockRandomised; + private int origPtr; + + private int nInUse; + private int nMTF; + + private static readonly int SETMASK = (1 << 21); + private static readonly int CLEARMASK = (~SETMASK); + private static readonly byte GREATER_ICOST = 15; + private static readonly byte LESSER_ICOST = 0; + private static readonly int SMALL_THRESH = 20; + private static readonly int DEPTH_THRESH = 10; + private static readonly int WORK_FACTOR = 30; + + /** + * Knuth's increments seem to work better than Incerpi-Sedgewick here. + * Possibly because the number of elems to sort is usually small, typically + * <= 20. + */ + private static readonly int[] increments = { 1, 4, 13, 40, 121, 364, 1093, 3280, + 9841, 29524, 88573, 265720, 797161, + 2391484 }; + + /// + /// BZip2Compressor writes its compressed data out via a BitWriter. This + /// is necessary because BZip2 does byte shredding. + /// + public BZip2Compressor(BitWriter writer) + : this(writer, BZip2.MaxBlockSize) + { + } + + public BZip2Compressor(BitWriter writer, int blockSize) + { + this.blockSize100k = blockSize; + this.bw = writer; + + // 20 provides a margin of slop (not to say "Safety"). The maximum + // size of an encoded run in the output block is 5 bytes, so really, 5 + // bytes ought to do, but this is a margin of slop found in the + // original bzip code. Not sure if important for decoding + // (decompressing). So we'll leave the slop. + this.outBlockFillThreshold = (blockSize * BZip2.BlockSizeMultiple) - 20; + this.cstate = new CompressionState(blockSize); + Reset(); + } + + + void Reset() + { + // initBlock(); + this.crc.Reset(); + this.currentByte = -1; + this.runLength = 0; + this.last = -1; + for (int i = 256; --i >= 0;) + this.cstate.inUse[i] = false; + //bw.Reset(); xxx? want this? no no no + } + + + public int BlockSize + { + get { return this.blockSize100k; } + } + + public uint Crc32 + { + get; private set; + } + + public int AvailableBytesOut + { + get; private set; + } + + /// + /// The number of uncompressed bytes being held in the buffer. + /// + /// + /// + /// I am thinking this may be useful in a Stream that uses this + /// compressor class. In the Close() method on the stream it could + /// check this value to see if anything has been written at all. You + /// may think the stream could easily track the number of bytes it + /// wrote, which would eliminate the need for this. But, there is the + /// case where the stream writes a complete block, and it is full, and + /// then writes no more. In that case the stream may want to check. + /// + /// + public int UncompressedBytes + { + get { return this.last + 1; } + } + + + /// + /// Accept new bytes into the compressor data buffer + /// + /// + /// + /// This method does the first-level (cheap) run-length encoding, and + /// stores the encoded data into the rle block. + /// + /// + public int Fill(byte[] buffer, int offset, int count) + { + if (this.last >= this.outBlockFillThreshold) + return 0; // We're full, I tell you! + + int bytesWritten = 0; + int limit = offset + count; + int rc; + + // do run-length-encoding until block is full + do + { + rc = write0(buffer[offset++]); + if (rc > 0) bytesWritten++; + } while (offset < limit && rc == 1); + + return bytesWritten; + } + + + + /// + /// Process one input byte into the block. + /// + /// + /// + /// + /// To "process" the byte means to do the run-length encoding. + /// There are 3 possible return values: + /// + /// 0 - the byte was not written, in other words, not + /// encoded into the block. This happens when the + /// byte b would require the start of a new run, and + /// the block has no more room for new runs. + /// + /// 1 - the byte was written, and the block is not full. + /// + /// 2 - the byte was written, and the block is full. + /// + /// + /// + /// 0 if the byte was not written, non-zero if written. + private int write0(byte b) + { + bool rc; + // there is no current run in progress + if (this.currentByte == -1) + { + this.currentByte = b; + this.runLength++; + return 1; + } + + // this byte is the same as the current run in progress + if (this.currentByte == b) + { + if (++this.runLength > 254) + { + rc = AddRunToOutputBlock(false); + this.currentByte = -1; + this.runLength = 0; + return (rc) ? 2 : 1; + } + return 1; // not full + } + + // This byte requires a new run. + // Put the prior run into the Run-length-encoded block, + // and try to start a new run. + rc = AddRunToOutputBlock(false); + + if (rc) + { + this.currentByte = -1; + this.runLength = 0; + // returning 0 implies the block is full, and the byte was not written. + return 0; + } + + // start a new run + this.runLength = 1; + this.currentByte = b; + return 1; + } + + /// + /// Append one run to the output block. + /// + /// + /// + /// + /// This compressor does run-length-encoding before BWT and etc. This + /// method simply appends a run to the output block. The append always + /// succeeds. The return value indicates whether the block is full: + /// false (not full) implies that at least one additional run could be + /// processed. + /// + /// + /// true if the block is now full; otherwise false. + private bool AddRunToOutputBlock(bool final) + { + runs++; + /* add_pair_to_block ( EState* s ) */ + int previousLast = this.last; + + // sanity check only - because of the check done at the + // bottom of this method, and the logic in write0(), this + // should never ever happen. + if (previousLast >= this.outBlockFillThreshold && !final) + { + var msg = String.Format("block overrun(final={2}): {0} >= threshold ({1})", + previousLast, this.outBlockFillThreshold, final); + throw new Exception(msg); + } + + // NB: the index used here into block is always (last+2). This is + // because last is -1 based - the initial value is -1, a flag value + // used to indicate that nothing has yet been written into the + // block. The endBlock() fn tests for -1 to detect empty blocks. Also, + // the first byte of block is used, during sorting, to hold block[last + // +1], which is the final byte value that had been written into the + // rle block. For those two reasons, the base offset from last is + // always +2. + + byte b = (byte)this.currentByte; + byte[] block = this.cstate.block; + this.cstate.inUse[b] = true; + int rl = this.runLength; + this.crc.UpdateCRC(b, rl); + + switch (rl) + { + case 1: + block[previousLast + 2] = b; + this.last = previousLast + 1; + break; + + case 2: + block[previousLast + 2] = b; + block[previousLast + 3] = b; + this.last = previousLast + 2; + break; + + case 3: + block[previousLast + 2] = b; + block[previousLast + 3] = b; + block[previousLast + 4] = b; + this.last = previousLast + 3; + break; + + default: + rl -= 4; + this.cstate.inUse[rl] = true; + block[previousLast + 2] = b; + block[previousLast + 3] = b; + block[previousLast + 4] = b; + block[previousLast + 5] = b; + block[previousLast + 6] = (byte)rl; + this.last = previousLast + 5; + break; + } + + // is full? + return (this.last >= this.outBlockFillThreshold); + } + + + /// + /// Compress the data that has been placed (Run-length-encoded) into the + /// block. The compressed data goes into the CompressedBytes array. + /// + /// + /// + /// Side effects: 1. fills the CompressedBytes array. 2. sets the + /// AvailableBytesOut property. + /// + /// + public void CompressAndWrite() // endBlock + { + if (this.runLength > 0) + AddRunToOutputBlock(true); + + this.currentByte = -1; + + // Console.WriteLine(" BZip2Compressor:CompressAndWrite (r={0} bcrc={1:X8})", + // runs, this.crc.Crc32Result); + + // has any data been written? + if (this.last == -1) + return; // no data; nothing to compress + + /* sort the block and establish posn of original string */ + blockSort(); + + /* + * A 6-byte block header, the value chosen arbitrarily as 0x314159265359 + * :-). A 32 bit value does not really give a strong enough guarantee + * that the value will not appear by chance in the compressed + * datastream. Worst-case probability of this event, for a 900k block, + * is about 2.0e-3 for 32 bits, 1.0e-5 for 40 bits and 4.0e-8 for 48 + * bits. For a compressed file of size 100Gb -- about 100000 blocks -- + * only a 48-bit marker will do. NB: normal compression/ decompression + * donot rely on these statistical properties. They are only important + * when trying to recover blocks from damaged files. + */ + this.bw.WriteByte(0x31); + this.bw.WriteByte(0x41); + this.bw.WriteByte(0x59); + this.bw.WriteByte(0x26); + this.bw.WriteByte(0x53); + this.bw.WriteByte(0x59); + + this.Crc32 = (uint)this.crc.Crc32Result; + this.bw.WriteInt(this.Crc32); + + /* Now a single bit indicating randomisation. */ + this.bw.WriteBits(1, (this.blockRandomised) ? 1U : 0U); + + /* Finally, block's contents proper. */ + moveToFrontCodeAndSend(); + + Reset(); + } + + + private void randomiseBlock() + { + bool[] inUse = this.cstate.inUse; + byte[] block = this.cstate.block; + int lastShadow = this.last; + + for (int i = 256; --i >= 0;) + inUse[i] = false; + + int rNToGo = 0; + int rTPos = 0; + for (int i = 0, j = 1; i <= lastShadow; i = j, j++) + { + if (rNToGo == 0) + { + rNToGo = (char)Rand.Rnums(rTPos); + if (++rTPos == 512) + { + rTPos = 0; + } + } + + rNToGo--; + block[j] ^= (byte)((rNToGo == 1) ? 1 : 0); + + // handle 16 bit signed numbers + inUse[block[j] & 0xff] = true; + } + + this.blockRandomised = true; + } + + private void mainSort() + { + CompressionState dataShadow = this.cstate; + int[] runningOrder = dataShadow.mainSort_runningOrder; + int[] copy = dataShadow.mainSort_copy; + bool[] bigDone = dataShadow.mainSort_bigDone; + int[] ftab = dataShadow.ftab; + byte[] block = dataShadow.block; + int[] fmap = dataShadow.fmap; + char[] quadrant = dataShadow.quadrant; + int lastShadow = this.last; + int workLimitShadow = this.workLimit; + bool firstAttemptShadow = this.firstAttempt; + + // Set up the 2-byte frequency table + for (int i = 65537; --i >= 0;) + { + ftab[i] = 0; + } + + /* + * In the various block-sized structures, live data runs from 0 to + * last+NUM_OVERSHOOT_BYTES inclusive. First, set up the overshoot area + * for block. + */ + for (int i = 0; i < BZip2.NUM_OVERSHOOT_BYTES; i++) + { + block[lastShadow + i + 2] = block[(i % (lastShadow + 1)) + 1]; + } + for (int i = lastShadow + BZip2.NUM_OVERSHOOT_BYTES + 1; --i >= 0;) + { + quadrant[i] = '\0'; + } + block[0] = block[lastShadow + 1]; + + // Complete the initial radix sort: + + int c1 = block[0] & 0xff; + for (int i = 0; i <= lastShadow; i++) + { + int c2 = block[i + 1] & 0xff; + ftab[(c1 << 8) + c2]++; + c1 = c2; + } + + for (int i = 1; i <= 65536; i++) + ftab[i] += ftab[i - 1]; + + c1 = block[1] & 0xff; + for (int i = 0; i < lastShadow; i++) + { + int c2 = block[i + 2] & 0xff; + fmap[--ftab[(c1 << 8) + c2]] = i; + c1 = c2; + } + + fmap[--ftab[((block[lastShadow + 1] & 0xff) << 8) + (block[1] & 0xff)]] = lastShadow; + + /* + * Now ftab contains the first loc of every small bucket. Calculate the + * running order, from smallest to largest big bucket. + */ + for (int i = 256; --i >= 0;) + { + bigDone[i] = false; + runningOrder[i] = i; + } + + for (int h = 364; h != 1;) + { + h /= 3; + for (int i = h; i <= 255; i++) + { + int vv = runningOrder[i]; + int a = ftab[(vv + 1) << 8] - ftab[vv << 8]; + int b = h - 1; + int j = i; + for (int ro = runningOrder[j - h]; (ftab[(ro + 1) << 8] - ftab[ro << 8]) > a; ro = runningOrder[j + - h]) + { + runningOrder[j] = ro; + j -= h; + if (j <= b) + { + break; + } + } + runningOrder[j] = vv; + } + } + + /* + * The main sorting loop. + */ + for (int i = 0; i <= 255; i++) + { + /* + * Process big buckets, starting with the least full. + */ + int ss = runningOrder[i]; + + // Step 1: + /* + * Complete the big bucket [ss] by quicksorting any unsorted small + * buckets [ss, j]. Hopefully previous pointer-scanning phases have + * already completed many of the small buckets [ss, j], so we don't + * have to sort them at all. + */ + for (int j = 0; j <= 255; j++) + { + int sb = (ss << 8) + j; + int ftab_sb = ftab[sb]; + if ((ftab_sb & SETMASK) != SETMASK) + { + int lo = ftab_sb & CLEARMASK; + int hi = (ftab[sb + 1] & CLEARMASK) - 1; + if (hi > lo) + { + mainQSort3(dataShadow, lo, hi, 2); + if (firstAttemptShadow + && (this.workDone > workLimitShadow)) + { + return; + } + } + ftab[sb] = ftab_sb | SETMASK; + } + } + + // Step 2: + // Now scan this big bucket so as to synthesise the + // sorted order for small buckets [t, ss] for all t != ss. + + for (int j = 0; j <= 255; j++) + { + copy[j] = ftab[(j << 8) + ss] & CLEARMASK; + } + + for (int j = ftab[ss << 8] & CLEARMASK, hj = (ftab[(ss + 1) << 8] & CLEARMASK); j < hj; j++) + { + int fmap_j = fmap[j]; + c1 = block[fmap_j] & 0xff; + if (!bigDone[c1]) + { + fmap[copy[c1]] = (fmap_j == 0) ? lastShadow : (fmap_j - 1); + copy[c1]++; + } + } + + for (int j = 256; --j >= 0;) + ftab[(j << 8) + ss] |= SETMASK; + + // Step 3: + /* + * The ss big bucket is now done. Record this fact, and update the + * quadrant descriptors. Remember to update quadrants in the + * overshoot area too, if necessary. The "if (i < 255)" test merely + * skips this updating for the last bucket processed, since updating + * for the last bucket is pointless. + */ + bigDone[ss] = true; + + if (i < 255) + { + int bbStart = ftab[ss << 8] & CLEARMASK; + int bbSize = (ftab[(ss + 1) << 8] & CLEARMASK) - bbStart; + int shifts = 0; + + while ((bbSize >> shifts) > 65534) + { + shifts++; + } + + for (int j = 0; j < bbSize; j++) + { + int a2update = fmap[bbStart + j]; + char qVal = (char)(j >> shifts); + quadrant[a2update] = qVal; + if (a2update < BZip2.NUM_OVERSHOOT_BYTES) + { + quadrant[a2update + lastShadow + 1] = qVal; + } + } + } + + } + } + + + private void blockSort() + { + this.workLimit = WORK_FACTOR * this.last; + this.workDone = 0; + this.blockRandomised = false; + this.firstAttempt = true; + mainSort(); + + if (this.firstAttempt && (this.workDone > this.workLimit)) + { + randomiseBlock(); + this.workLimit = this.workDone = 0; + this.firstAttempt = false; + mainSort(); + } + + int[] fmap = this.cstate.fmap; + this.origPtr = -1; + for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) + { + if (fmap[i] == 0) + { + this.origPtr = i; + break; + } + } + + // assert (this.origPtr != -1) : this.origPtr; + } + + + /** + * This is the most hammered method of this class. + * + *

+ * This is the version using unrolled loops. + *

+ */ + private bool mainSimpleSort(CompressionState dataShadow, int lo, + int hi, int d) + { + int bigN = hi - lo + 1; + if (bigN < 2) + { + return this.firstAttempt && (this.workDone > this.workLimit); + } + + int hp = 0; + while (increments[hp] < bigN) + hp++; + + int[] fmap = dataShadow.fmap; + char[] quadrant = dataShadow.quadrant; + byte[] block = dataShadow.block; + int lastShadow = this.last; + int lastPlus1 = lastShadow + 1; + bool firstAttemptShadow = this.firstAttempt; + int workLimitShadow = this.workLimit; + int workDoneShadow = this.workDone; + + // Following block contains unrolled code which could be shortened by + // coding it in additional loops. + + // HP: + while (--hp >= 0) + { + int h = increments[hp]; + int mj = lo + h - 1; + + for (int i = lo + h; i <= hi;) + { + // copy + for (int k = 3; (i <= hi) && (--k >= 0); i++) + { + int v = fmap[i]; + int vd = v + d; + int j = i; + + // for (int a; + // (j > mj) && mainGtU((a = fmap[j - h]) + d, vd, + // block, quadrant, lastShadow); + // j -= h) { + // fmap[j] = a; + // } + // + // unrolled version: + + // start inline mainGTU + bool onceRunned = false; + int a = 0; + + HAMMER: while (true) + { + if (onceRunned) + { + fmap[j] = a; + if ((j -= h) <= mj) + { + goto END_HAMMER; + } + } + else + { + onceRunned = true; + } + + a = fmap[j - h]; + int i1 = a + d; + int i2 = vd; + + // following could be done in a loop, but + // unrolled it for performance: + if (block[i1 + 1] == block[i2 + 1]) + { + if (block[i1 + 2] == block[i2 + 2]) + { + if (block[i1 + 3] == block[i2 + 3]) + { + if (block[i1 + 4] == block[i2 + 4]) + { + if (block[i1 + 5] == block[i2 + 5]) + { + if (block[(i1 += 6)] == block[(i2 += 6)]) + { + int x = lastShadow; + X: while (x > 0) + { + x -= 4; + + if (block[i1 + 1] == block[i2 + 1]) + { + if (quadrant[i1] == quadrant[i2]) + { + if (block[i1 + 2] == block[i2 + 2]) + { + if (quadrant[i1 + 1] == quadrant[i2 + 1]) + { + if (block[i1 + 3] == block[i2 + 3]) + { + if (quadrant[i1 + 2] == quadrant[i2 + 2]) + { + if (block[i1 + 4] == block[i2 + 4]) + { + if (quadrant[i1 + 3] == quadrant[i2 + 3]) + { + if ((i1 += 4) >= lastPlus1) + { + i1 -= lastPlus1; + } + if ((i2 += 4) >= lastPlus1) + { + i2 -= lastPlus1; + } + workDoneShadow++; + goto X; + } + else if ((quadrant[i1 + 3] > quadrant[i2 + 3])) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((quadrant[i1 + 2] > quadrant[i2 + 2])) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((quadrant[i1 + 1] > quadrant[i2 + 1])) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((quadrant[i1] > quadrant[i2])) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + + } + goto END_HAMMER; + } // while x > 0 + else + { + if ((block[i1] & 0xff) > (block[i2] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + } + else if ((block[i1 + 5] & 0xff) > (block[i2 + 5] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + } + else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) + { + goto HAMMER; + } + else + { + goto END_HAMMER; + } + + } // HAMMER + + END_HAMMER: + // end inline mainGTU + + fmap[j] = v; + } + + if (firstAttemptShadow && (i <= hi) + && (workDoneShadow > workLimitShadow)) + { + goto END_HP; + } + } + } + END_HP: + + this.workDone = workDoneShadow; + return firstAttemptShadow && (workDoneShadow > workLimitShadow); + } + + + + private static void vswap(int[] fmap, int p1, int p2, int n) + { + n += p1; + while (p1 < n) + { + int t = fmap[p1]; + fmap[p1++] = fmap[p2]; + fmap[p2++] = t; + } + } + + private static byte med3(byte a, byte b, byte c) + { + return (a < b) ? (b < c ? b : a < c ? c : a) : (b > c ? b : a > c ? c + : a); + } + + + /** + * Method "mainQSort3", file "blocksort.c", BZip2 1.0.2 + */ + private void mainQSort3(CompressionState dataShadow, int loSt, + int hiSt, int dSt) + { + int[] stack_ll = dataShadow.stack_ll; + int[] stack_hh = dataShadow.stack_hh; + int[] stack_dd = dataShadow.stack_dd; + int[] fmap = dataShadow.fmap; + byte[] block = dataShadow.block; + + stack_ll[0] = loSt; + stack_hh[0] = hiSt; + stack_dd[0] = dSt; + + for (int sp = 1; --sp >= 0;) + { + int lo = stack_ll[sp]; + int hi = stack_hh[sp]; + int d = stack_dd[sp]; + + if ((hi - lo < SMALL_THRESH) || (d > DEPTH_THRESH)) + { + if (mainSimpleSort(dataShadow, lo, hi, d)) + { + return; + } + } + else + { + int d1 = d + 1; + int med = med3(block[fmap[lo] + d1], + block[fmap[hi] + d1], block[fmap[(lo + hi) >> 1] + d1]) & 0xff; + + int unLo = lo; + int unHi = hi; + int ltLo = lo; + int gtHi = hi; + + while (true) + { + while (unLo <= unHi) + { + int n = (block[fmap[unLo] + d1] & 0xff) + - med; + if (n == 0) + { + int temp = fmap[unLo]; + fmap[unLo++] = fmap[ltLo]; + fmap[ltLo++] = temp; + } + else if (n < 0) + { + unLo++; + } + else + { + break; + } + } + + while (unLo <= unHi) + { + int n = (block[fmap[unHi] + d1] & 0xff) + - med; + if (n == 0) + { + int temp = fmap[unHi]; + fmap[unHi--] = fmap[gtHi]; + fmap[gtHi--] = temp; + } + else if (n > 0) + { + unHi--; + } + else + { + break; + } + } + + if (unLo <= unHi) + { + int temp = fmap[unLo]; + fmap[unLo++] = fmap[unHi]; + fmap[unHi--] = temp; + } + else + { + break; + } + } + + if (gtHi < ltLo) + { + stack_ll[sp] = lo; + stack_hh[sp] = hi; + stack_dd[sp] = d1; + sp++; + } + else + { + int n = ((ltLo - lo) < (unLo - ltLo)) ? (ltLo - lo) + : (unLo - ltLo); + vswap(fmap, lo, unLo - n, n); + int m = ((hi - gtHi) < (gtHi - unHi)) ? (hi - gtHi) + : (gtHi - unHi); + vswap(fmap, unLo, hi - m + 1, m); + + n = lo + unLo - ltLo - 1; + m = hi - (gtHi - unHi) + 1; + + stack_ll[sp] = lo; + stack_hh[sp] = n; + stack_dd[sp] = d; + sp++; + + stack_ll[sp] = n + 1; + stack_hh[sp] = m - 1; + stack_dd[sp] = d1; + sp++; + + stack_ll[sp] = m; + stack_hh[sp] = hi; + stack_dd[sp] = d; + sp++; + } + } + } + } + + + + private void generateMTFValues() + { + int lastShadow = this.last; + CompressionState dataShadow = this.cstate; + bool[] inUse = dataShadow.inUse; + byte[] block = dataShadow.block; + int[] fmap = dataShadow.fmap; + char[] sfmap = dataShadow.sfmap; + int[] mtfFreq = dataShadow.mtfFreq; + byte[] unseqToSeq = dataShadow.unseqToSeq; + byte[] yy = dataShadow.generateMTFValues_yy; + + // make maps + int nInUseShadow = 0; + for (int i = 0; i < 256; i++) + { + if (inUse[i]) + { + unseqToSeq[i] = (byte)nInUseShadow; + nInUseShadow++; + } + } + this.nInUse = nInUseShadow; + + int eob = nInUseShadow + 1; + + for (int i = eob; i >= 0; i--) + { + mtfFreq[i] = 0; + } + + for (int i = nInUseShadow; --i >= 0;) + { + yy[i] = (byte)i; + } + + int wr = 0; + int zPend = 0; + + for (int i = 0; i <= lastShadow; i++) + { + byte ll_i = unseqToSeq[block[fmap[i]] & 0xff]; + byte tmp = yy[0]; + int j = 0; + + while (ll_i != tmp) + { + j++; + byte tmp2 = tmp; + tmp = yy[j]; + yy[j] = tmp2; + } + yy[0] = tmp; + + if (j == 0) + { + zPend++; + } + else + { + if (zPend > 0) + { + zPend--; + while (true) + { + if ((zPend & 1) == 0) + { + sfmap[wr] = BZip2.RUNA; + wr++; + mtfFreq[BZip2.RUNA]++; + } + else + { + sfmap[wr] = BZip2.RUNB; + wr++; + mtfFreq[BZip2.RUNB]++; + } + + if (zPend >= 2) + { + zPend = (zPend - 2) >> 1; + } + else + { + break; + } + } + zPend = 0; + } + sfmap[wr] = (char)(j + 1); + wr++; + mtfFreq[j + 1]++; + } + } + + if (zPend > 0) + { + zPend--; + while (true) + { + if ((zPend & 1) == 0) + { + sfmap[wr] = BZip2.RUNA; + wr++; + mtfFreq[BZip2.RUNA]++; + } + else + { + sfmap[wr] = BZip2.RUNB; + wr++; + mtfFreq[BZip2.RUNB]++; + } + + if (zPend >= 2) + { + zPend = (zPend - 2) >> 1; + } + else + { + break; + } + } + } + + sfmap[wr] = (char)eob; + mtfFreq[eob]++; + this.nMTF = wr + 1; + } + + + private static void hbAssignCodes(int[] code, byte[] length, + int minLen, int maxLen, + int alphaSize) + { + int vec = 0; + for (int n = minLen; n <= maxLen; n++) + { + for (int i = 0; i < alphaSize; i++) + { + if ((length[i] & 0xff) == n) + { + code[i] = vec; + vec++; + } + } + vec <<= 1; + } + } + + + + + private void sendMTFValues() + { + byte[][] len = this.cstate.sendMTFValues_len; + int alphaSize = this.nInUse + 2; + + for (int t = BZip2.NGroups; --t >= 0;) + { + byte[] len_t = len[t]; + for (int v = alphaSize; --v >= 0;) + { + len_t[v] = GREATER_ICOST; + } + } + + /* Decide how many coding tables to use */ + // assert (this.nMTF > 0) : this.nMTF; + int nGroups = (this.nMTF < 200) ? 2 : (this.nMTF < 600) ? 3 + : (this.nMTF < 1200) ? 4 : (this.nMTF < 2400) ? 5 : 6; + + /* Generate an initial set of coding tables */ + sendMTFValues0(nGroups, alphaSize); + + /* + * Iterate up to N_ITERS times to improve the tables. + */ + int nSelectors = sendMTFValues1(nGroups, alphaSize); + + /* Compute MTF values for the selectors. */ + sendMTFValues2(nGroups, nSelectors); + + /* Assign actual codes for the tables. */ + sendMTFValues3(nGroups, alphaSize); + + /* Transmit the mapping table. */ + sendMTFValues4(); + + /* Now the selectors. */ + sendMTFValues5(nGroups, nSelectors); + + /* Now the coding tables. */ + sendMTFValues6(nGroups, alphaSize); + + /* And finally, the block data proper */ + sendMTFValues7(nSelectors); + } + + private void sendMTFValues0(int nGroups, int alphaSize) + { + byte[][] len = this.cstate.sendMTFValues_len; + int[] mtfFreq = this.cstate.mtfFreq; + + int remF = this.nMTF; + int gs = 0; + + for (int nPart = nGroups; nPart > 0; nPart--) + { + int tFreq = remF / nPart; + int ge = gs - 1; + int aFreq = 0; + + for (int a = alphaSize - 1; (aFreq < tFreq) && (ge < a);) + { + aFreq += mtfFreq[++ge]; + } + + if ((ge > gs) && (nPart != nGroups) && (nPart != 1) + && (((nGroups - nPart) & 1) != 0)) + { + aFreq -= mtfFreq[ge--]; + } + + byte[] len_np = len[nPart - 1]; + for (int v = alphaSize; --v >= 0;) + { + if ((v >= gs) && (v <= ge)) + { + len_np[v] = LESSER_ICOST; + } + else + { + len_np[v] = GREATER_ICOST; + } + } + + gs = ge + 1; + remF -= aFreq; + } + } + + + private static void hbMakeCodeLengths(byte[] len, int[] freq, + CompressionState state1, int alphaSize, + int maxLen) + { + /* + * Nodes and heap entries run from 1. Entry 0 for both the heap and + * nodes is a sentinel. + */ + int[] heap = state1.heap; + int[] weight = state1.weight; + int[] parent = state1.parent; + + for (int i = alphaSize; --i >= 0;) + { + weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8; + } + + for (bool tooLong = true; tooLong;) + { + tooLong = false; + + int nNodes = alphaSize; + int nHeap = 0; + heap[0] = 0; + weight[0] = 0; + parent[0] = -2; + + for (int i = 1; i <= alphaSize; i++) + { + parent[i] = -1; + nHeap++; + heap[nHeap] = i; + + int zz = nHeap; + int tmp = heap[zz]; + while (weight[tmp] < weight[heap[zz >> 1]]) + { + heap[zz] = heap[zz >> 1]; + zz >>= 1; + } + heap[zz] = tmp; + } + + while (nHeap > 1) + { + int n1 = heap[1]; + heap[1] = heap[nHeap]; + nHeap--; + + int yy = 0; + int zz = 1; + int tmp = heap[1]; + + while (true) + { + yy = zz << 1; + + if (yy > nHeap) + { + break; + } + + if ((yy < nHeap) + && (weight[heap[yy + 1]] < weight[heap[yy]])) + { + yy++; + } + + if (weight[tmp] < weight[heap[yy]]) + { + break; + } + + heap[zz] = heap[yy]; + zz = yy; + } + + heap[zz] = tmp; + + int n2 = heap[1]; + heap[1] = heap[nHeap]; + nHeap--; + + yy = 0; + zz = 1; + tmp = heap[1]; + + while (true) + { + yy = zz << 1; + + if (yy > nHeap) + { + break; + } + + if ((yy < nHeap) + && (weight[heap[yy + 1]] < weight[heap[yy]])) + { + yy++; + } + + if (weight[tmp] < weight[heap[yy]]) + { + break; + } + + heap[zz] = heap[yy]; + zz = yy; + } + + heap[zz] = tmp; + nNodes++; + parent[n1] = parent[n2] = nNodes; + + int weight_n1 = weight[n1]; + int weight_n2 = weight[n2]; + weight[nNodes] = (int)(((uint)weight_n1 & 0xffffff00U) + + ((uint)weight_n2 & 0xffffff00U)) + | (1 + (((weight_n1 & 0x000000ff) + > (weight_n2 & 0x000000ff)) + ? (weight_n1 & 0x000000ff) + : (weight_n2 & 0x000000ff))); + + parent[nNodes] = -1; + nHeap++; + heap[nHeap] = nNodes; + + tmp = 0; + zz = nHeap; + tmp = heap[zz]; + int weight_tmp = weight[tmp]; + while (weight_tmp < weight[heap[zz >> 1]]) + { + heap[zz] = heap[zz >> 1]; + zz >>= 1; + } + heap[zz] = tmp; + + } + + for (int i = 1; i <= alphaSize; i++) + { + int j = 0; + int k = i; + + for (int parent_k; (parent_k = parent[k]) >= 0;) + { + k = parent_k; + j++; + } + + len[i - 1] = (byte)j; + if (j > maxLen) + { + tooLong = true; + } + } + + if (tooLong) + { + for (int i = 1; i < alphaSize; i++) + { + int j = weight[i] >> 8; + j = 1 + (j >> 1); + weight[i] = j << 8; + } + } + } + } + + + private int sendMTFValues1(int nGroups, int alphaSize) + { + CompressionState dataShadow = this.cstate; + int[][] rfreq = dataShadow.sendMTFValues_rfreq; + int[] fave = dataShadow.sendMTFValues_fave; + short[] cost = dataShadow.sendMTFValues_cost; + char[] sfmap = dataShadow.sfmap; + byte[] selector = dataShadow.selector; + byte[][] len = dataShadow.sendMTFValues_len; + byte[] len_0 = len[0]; + byte[] len_1 = len[1]; + byte[] len_2 = len[2]; + byte[] len_3 = len[3]; + byte[] len_4 = len[4]; + byte[] len_5 = len[5]; + int nMTFShadow = this.nMTF; + + int nSelectors = 0; + + for (int iter = 0; iter < BZip2.N_ITERS; iter++) + { + for (int t = nGroups; --t >= 0;) + { + fave[t] = 0; + int[] rfreqt = rfreq[t]; + for (int i = alphaSize; --i >= 0;) + { + rfreqt[i] = 0; + } + } + + nSelectors = 0; + + for (int gs = 0; gs < this.nMTF;) + { + /* Set group start & end marks. */ + + /* + * Calculate the cost of this group as coded by each of the + * coding tables. + */ + + int ge = Math.Min(gs + BZip2.G_SIZE - 1, nMTFShadow - 1); + + if (nGroups == BZip2.NGroups) + { + // unrolled version of the else-block + + int[] c = new int[6]; + + for (int i = gs; i <= ge; i++) + { + int icv = sfmap[i]; + c[0] += len_0[icv] & 0xff; + c[1] += len_1[icv] & 0xff; + c[2] += len_2[icv] & 0xff; + c[3] += len_3[icv] & 0xff; + c[4] += len_4[icv] & 0xff; + c[5] += len_5[icv] & 0xff; + } + + cost[0] = (short)c[0]; + cost[1] = (short)c[1]; + cost[2] = (short)c[2]; + cost[3] = (short)c[3]; + cost[4] = (short)c[4]; + cost[5] = (short)c[5]; + } + else + { + for (int t = nGroups; --t >= 0;) + { + cost[t] = 0; + } + + for (int i = gs; i <= ge; i++) + { + int icv = sfmap[i]; + for (int t = nGroups; --t >= 0;) + { + cost[t] += (short)(len[t][icv] & 0xff); + } + } + } + + /* + * Find the coding table which is best for this group, and + * record its identity in the selector table. + */ + int bt = -1; + for (int t = nGroups, bc = 999999999; --t >= 0;) + { + int cost_t = cost[t]; + if (cost_t < bc) + { + bc = cost_t; + bt = t; + } + } + + fave[bt]++; + selector[nSelectors] = (byte)bt; + nSelectors++; + + /* + * Increment the symbol frequencies for the selected table. + */ + int[] rfreq_bt = rfreq[bt]; + for (int i = gs; i <= ge; i++) + { + rfreq_bt[sfmap[i]]++; + } + + gs = ge + 1; + } + + /* + * Recompute the tables based on the accumulated frequencies. + */ + for (int t = 0; t < nGroups; t++) + { + hbMakeCodeLengths(len[t], rfreq[t], this.cstate, alphaSize, 20); + } + } + + return nSelectors; + } + + private void sendMTFValues2(int nGroups, int nSelectors) + { + // assert (nGroups < 8) : nGroups; + + CompressionState dataShadow = this.cstate; + byte[] pos = dataShadow.sendMTFValues2_pos; + + for (int i = nGroups; --i >= 0;) + { + pos[i] = (byte)i; + } + + for (int i = 0; i < nSelectors; i++) + { + byte ll_i = dataShadow.selector[i]; + byte tmp = pos[0]; + int j = 0; + + while (ll_i != tmp) + { + j++; + byte tmp2 = tmp; + tmp = pos[j]; + pos[j] = tmp2; + } + + pos[0] = tmp; + dataShadow.selectorMtf[i] = (byte)j; + } + } + + private void sendMTFValues3(int nGroups, int alphaSize) + { + int[][] code = this.cstate.sendMTFValues_code; + byte[][] len = this.cstate.sendMTFValues_len; + + for (int t = 0; t < nGroups; t++) + { + int minLen = 32; + int maxLen = 0; + byte[] len_t = len[t]; + for (int i = alphaSize; --i >= 0;) + { + int l = len_t[i] & 0xff; + if (l > maxLen) + { + maxLen = l; + } + if (l < minLen) + { + minLen = l; + } + } + + // assert (maxLen <= 20) : maxLen; + // assert (minLen >= 1) : minLen; + + hbAssignCodes(code[t], len[t], minLen, maxLen, alphaSize); + } + } + + private void sendMTFValues4() + { + bool[] inUse = this.cstate.inUse; + bool[] inUse16 = this.cstate.sentMTFValues4_inUse16; + + for (int i = 16; --i >= 0;) + { + inUse16[i] = false; + int i16 = i * 16; + for (int j = 16; --j >= 0;) + { + if (inUse[i16 + j]) + { + inUse16[i] = true; + } + } + } + + uint u = 0; + for (int i = 0; i < 16; i++) + { + if (inUse16[i]) + u |= 1U << (16 - i - 1); + } + this.bw.WriteBits(16, u); + + + for (int i = 0; i < 16; i++) + { + if (inUse16[i]) + { + int i16 = i * 16; + u = 0; + for (int j = 0; j < 16; j++) + { + if (inUse[i16 + j]) + { + u |= 1U << (16 - j - 1); + } + } + this.bw.WriteBits(16, u); + } + } + } + + + private void sendMTFValues5(int nGroups, int nSelectors) + { + this.bw.WriteBits(3, (uint)nGroups); + this.bw.WriteBits(15, (uint)nSelectors); + + byte[] selectorMtf = this.cstate.selectorMtf; + + for (int i = 0; i < nSelectors; i++) + { + for (int j = 0, hj = selectorMtf[i] & 0xff; j < hj; j++) + { + this.bw.WriteBits(1, 1); + } + + this.bw.WriteBits(1, 0); + } + } + + private void sendMTFValues6(int nGroups, int alphaSize) + { + byte[][] len = this.cstate.sendMTFValues_len; + + for (int t = 0; t < nGroups; t++) + { + byte[] len_t = len[t]; + uint curr = (uint)(len_t[0] & 0xff); + this.bw.WriteBits(5, curr); + + for (int i = 0; i < alphaSize; i++) + { + int lti = len_t[i] & 0xff; + while (curr < lti) + { + this.bw.WriteBits(2, 2U); + curr++; /* 10 */ + } + + while (curr > lti) + { + this.bw.WriteBits(2, 3U); + curr--; /* 11 */ + } + + this.bw.WriteBits(1, 0U); + } + } + } + + + private void sendMTFValues7(int nSelectors) + { + byte[][] len = this.cstate.sendMTFValues_len; + int[][] code = this.cstate.sendMTFValues_code; + byte[] selector = this.cstate.selector; + char[] sfmap = this.cstate.sfmap; + int nMTFShadow = this.nMTF; + + int selCtr = 0; + + for (int gs = 0; gs < nMTFShadow;) + { + int ge = Math.Min(gs + BZip2.G_SIZE - 1, nMTFShadow - 1); + int ix = selector[selCtr] & 0xff; + int[] code_selCtr = code[ix]; + byte[] len_selCtr = len[ix]; + + while (gs <= ge) + { + int sfmap_i = sfmap[gs]; + int n = len_selCtr[sfmap_i] & 0xFF; + this.bw.WriteBits(n, (uint)code_selCtr[sfmap_i]); + gs++; + } + + gs = ge + 1; + selCtr++; + } + } + + private void moveToFrontCodeAndSend() + { + this.bw.WriteBits(24, (uint)this.origPtr); + generateMTFValues(); + sendMTFValues(); + } + + + + + + + private class CompressionState + { + // with blockSize 900k + public readonly bool[] inUse = new bool[256]; // 256 byte + public readonly byte[] unseqToSeq = new byte[256]; // 256 byte + public readonly int[] mtfFreq = new int[BZip2.MaxAlphaSize]; // 1032 byte + public readonly byte[] selector = new byte[BZip2.MaxSelectors]; // 18002 byte + public readonly byte[] selectorMtf = new byte[BZip2.MaxSelectors]; // 18002 byte + + public readonly byte[] generateMTFValues_yy = new byte[256]; // 256 byte + public byte[][] sendMTFValues_len; + + // byte + public int[][] sendMTFValues_rfreq; + + // byte + public readonly int[] sendMTFValues_fave = new int[BZip2.NGroups]; // 24 byte + public readonly short[] sendMTFValues_cost = new short[BZip2.NGroups]; // 12 byte + public int[][] sendMTFValues_code; + + // byte + public readonly byte[] sendMTFValues2_pos = new byte[BZip2.NGroups]; // 6 byte + public readonly bool[] sentMTFValues4_inUse16 = new bool[16]; // 16 byte + + public readonly int[] stack_ll = new int[BZip2.QSORT_STACK_SIZE]; // 4000 byte + public readonly int[] stack_hh = new int[BZip2.QSORT_STACK_SIZE]; // 4000 byte + public readonly int[] stack_dd = new int[BZip2.QSORT_STACK_SIZE]; // 4000 byte + + public readonly int[] mainSort_runningOrder = new int[256]; // 1024 byte + public readonly int[] mainSort_copy = new int[256]; // 1024 byte + public readonly bool[] mainSort_bigDone = new bool[256]; // 256 byte + + public int[] heap = new int[BZip2.MaxAlphaSize + 2]; // 1040 byte + public int[] weight = new int[BZip2.MaxAlphaSize * 2]; // 2064 byte + public int[] parent = new int[BZip2.MaxAlphaSize * 2]; // 2064 byte + + public readonly int[] ftab = new int[65537]; // 262148 byte + // ------------ + // 333408 byte + + public byte[] block; // 900021 byte + public int[] fmap; // 3600000 byte + public char[] sfmap; // 3600000 byte + + // ------------ + // 8433529 byte + // ============ + + /** + * Array instance identical to sfmap, both are used only + * temporarily and independently, so we do not need to allocate + * additional memory. + */ + public char[] quadrant; + + public CompressionState(int blockSize100k) + { + int n = blockSize100k * BZip2.BlockSizeMultiple; + this.block = new byte[(n + 1 + BZip2.NUM_OVERSHOOT_BYTES)]; + this.fmap = new int[n]; + this.sfmap = new char[2 * n]; + this.quadrant = this.sfmap; + this.sendMTFValues_len = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.sendMTFValues_rfreq = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.sendMTFValues_code = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + } + + } + + + + } +} \ No newline at end of file diff --git a/SabreTools.Compression/BZip2/BZip2InputStream.cs b/SabreTools.Compression/BZip2/BZip2InputStream.cs new file mode 100644 index 0000000..d871591 --- /dev/null +++ b/SabreTools.Compression/BZip2/BZip2InputStream.cs @@ -0,0 +1,1386 @@ +// BZip2InputStream.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-July-31 11:57:32> +// +// ------------------------------------------------------------------ +// +// This module defines the BZip2InputStream class, which is a decompressing +// stream that handles BZIP2. This code is derived from Apache commons source code. +// The license below applies to the original Apache code. +// +// ------------------------------------------------------------------ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * to whom the Ant project is very grateful for his + * great code. + */ + +// compile: msbuild +// not: csc.exe /t:library /debug+ /out:SabreTools.Compression.BZip2.dll BZip2InputStream.cs BCRC32.cs Rand.cs + +using System; +using System.IO; + +#nullable disable +namespace SabreTools.Compression.BZip2 +{ + /// + /// A read-only decorator stream that performs BZip2 decompression on Read. + /// + public class BZip2InputStream : System.IO.Stream + { + bool _disposed; + bool _leaveOpen; + Int64 totalBytesRead; + private int last; + + /* for undoing the Burrows-Wheeler transform */ + private int origPtr; + + // blockSize100k: 0 .. 9. + // + // This var name is a misnomer. The actual block size is 100000 + // * blockSize100k. (not 100k * blocksize100k) + private int blockSize100k; + private bool blockRandomised; + private int bsBuff; + private int bsLive; + private readonly CRC32 crc = new CRC32(true); + private int nInUse; + private Stream input; + private int currentChar = -1; + + /// + /// Compressor State + /// + enum CState + { + EOF = 0, + START_BLOCK = 1, + RAND_PART_A = 2, + RAND_PART_B = 3, + RAND_PART_C = 4, + NO_RAND_PART_A = 5, + NO_RAND_PART_B = 6, + NO_RAND_PART_C = 7, + } + + private CState currentState = CState.START_BLOCK; + + private uint storedBlockCRC, storedCombinedCRC; + private uint computedBlockCRC, computedCombinedCRC; + + // Variables used by setup* methods exclusively + private int su_count; + private int su_ch2; + private int su_chPrev; + private int su_i2; + private int su_j2; + private int su_rNToGo; + private int su_rTPos; + private int su_tPos; + private char su_z; + private BZip2InputStream.DecompressionState data; + + + /// + /// Create a BZip2InputStream, wrapping it around the given input Stream. + /// + /// + /// + /// The input stream will be closed when the BZip2InputStream is closed. + /// + /// + /// The stream from which to read compressed data + public BZip2InputStream(Stream input) + : this(input, false) + { } + + + /// + /// Create a BZip2InputStream with the given stream, and + /// specifying whether to leave the wrapped stream open when + /// the BZip2InputStream is closed. + /// + /// The stream from which to read compressed data + /// + /// Whether to leave the input stream open, when the BZip2InputStream closes. + /// + /// + /// + /// + /// This example reads a bzip2-compressed file, decompresses it, + /// and writes the decompressed data into a newly created file. + /// + /// + /// var fname = "logfile.log.bz2"; + /// using (var fs = File.OpenRead(fname)) + /// { + /// using (var decompressor = new SabreTools.Compression.BZip2.BZip2InputStream(fs)) + /// { + /// var outFname = fname + ".decompressed"; + /// using (var output = File.Create(outFname)) + /// { + /// byte[] buffer = new byte[2048]; + /// int n; + /// while ((n = decompressor.Read(buffer, 0, buffer.Length)) > 0) + /// { + /// output.Write(buffer, 0, n); + /// } + /// } + /// } + /// } + /// + /// + public BZip2InputStream(Stream input, bool leaveOpen) + : base() + { + + this.input = input; + this._leaveOpen = leaveOpen; + init(); + } + + /// + /// Read data from the stream. + /// + /// + /// + /// + /// To decompress a BZip2 data stream, create a BZip2InputStream, + /// providing a stream that reads compressed data. Then call Read() on + /// that BZip2InputStream, and the data read will be decompressed + /// as you read. + /// + /// + /// + /// A BZip2InputStream can be used only for Read(), not for Write(). + /// + /// + /// + /// The buffer into which the read data should be placed. + /// the offset within that data array to put the first byte read. + /// the number of bytes to read. + /// the number of bytes actually read + public override int Read(byte[] buffer, int offset, int count) + { + if (offset < 0) + throw new IndexOutOfRangeException(String.Format("offset ({0}) must be > 0", offset)); + + if (count < 0) + throw new IndexOutOfRangeException(String.Format("count ({0}) must be > 0", count)); + + if (offset + count > buffer.Length) + throw new IndexOutOfRangeException(String.Format("offset({0}) count({1}) bLength({2})", + offset, count, buffer.Length)); + + if (this.input == null) + throw new IOException("the stream is not open"); + + + int hi = offset + count; + int destOffset = offset; + for (int b; (destOffset < hi) && ((b = ReadByte()) >= 0);) + { + buffer[destOffset++] = (byte)b; + } + + return (destOffset == offset) ? -1 : (destOffset - offset); + } + + private void MakeMaps() + { + bool[] inUse = this.data.inUse; + byte[] seqToUnseq = this.data.seqToUnseq; + + int n = 0; + + for (int i = 0; i < 256; i++) + { + if (inUse[i]) + seqToUnseq[n++] = (byte)i; + } + + this.nInUse = n; + } + + /// + /// Read a single byte from the stream. + /// + /// the byte read from the stream, or -1 if EOF + public override int ReadByte() + { + int retChar = this.currentChar; + totalBytesRead++; + switch (this.currentState) + { + case CState.EOF: + return -1; + + case CState.START_BLOCK: + throw new IOException("bad state"); + + case CState.RAND_PART_A: + throw new IOException("bad state"); + + case CState.RAND_PART_B: + SetupRandPartB(); + break; + + case CState.RAND_PART_C: + SetupRandPartC(); + break; + + case CState.NO_RAND_PART_A: + throw new IOException("bad state"); + + case CState.NO_RAND_PART_B: + SetupNoRandPartB(); + break; + + case CState.NO_RAND_PART_C: + SetupNoRandPartC(); + break; + + default: + throw new IOException("bad state"); + } + + return retChar; + } + + + + + /// + /// Indicates whether the stream can be read. + /// + /// + /// The return value depends on whether the captive stream supports reading. + /// + public override bool CanRead + { + get + { + if (_disposed) throw new ObjectDisposedException("BZip2Stream"); + return this.input.CanRead; + } + } + + + /// + /// Indicates whether the stream supports Seek operations. + /// + /// + /// Always returns false. + /// + public override bool CanSeek + { + get { return false; } + } + + + /// + /// Indicates whether the stream can be written. + /// + /// + /// The return value depends on whether the captive stream supports writing. + /// + public override bool CanWrite + { + get + { + if (_disposed) throw new ObjectDisposedException("BZip2Stream"); + return input.CanWrite; + } + } + + /// + /// Flush the stream. + /// + public override void Flush() + { + if (_disposed) throw new ObjectDisposedException("BZip2Stream"); + input.Flush(); + } + + /// + /// Reading this property always throws a . + /// + public override long Length + { + get { throw new NotImplementedException(); } + } + + /// + /// The position of the stream pointer. + /// + /// + /// + /// Setting this property always throws a . Reading will return the + /// total number of uncompressed bytes read in. + /// + public override long Position + { + get + { + return this.totalBytesRead; + } + set { throw new NotImplementedException(); } + } + + /// + /// Calling this method always throws a . + /// + /// this is irrelevant, since it will always throw! + /// this is irrelevant, since it will always throw! + /// irrelevant! + public override long Seek(long offset, System.IO.SeekOrigin origin) + { + throw new NotImplementedException(); + } + + /// + /// Calling this method always throws a . + /// + /// this is irrelevant, since it will always throw! + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + /// + /// Calling this method always throws a . + /// + /// this parameter is never used + /// this parameter is never used + /// this parameter is never used + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + + /// + /// Dispose the stream. + /// + /// + /// indicates whether the Dispose method was invoked by user code. + /// + protected override void Dispose(bool disposing) + { + try + { + if (!_disposed) + { + if (disposing && (this.input != null)) + this.input.Close(); + _disposed = true; + } + } + finally + { + base.Dispose(disposing); + } + } + + + void init() + { + if (null == this.input) + throw new IOException("No input Stream"); + + if (!this.input.CanRead) + throw new IOException("Unreadable input Stream"); + + CheckMagicChar('B', 0); + CheckMagicChar('Z', 1); + CheckMagicChar('h', 2); + + int blockSize = this.input.ReadByte(); + + if ((blockSize < '1') || (blockSize > '9')) + throw new IOException("Stream is not BZip2 formatted: illegal " + + "blocksize " + (char)blockSize); + + this.blockSize100k = blockSize - '0'; + + InitBlock(); + SetupBlock(); + } + + + void CheckMagicChar(char expected, int position) + { + int magic = this.input.ReadByte(); + if (magic != (int)expected) + { + var msg = String.Format("Not a valid BZip2 stream. byte {0}, expected '{1}', got '{2}'", + position, (int)expected, magic); + throw new IOException(msg); + } + } + + + void InitBlock() + { + char magic0 = bsGetUByte(); + char magic1 = bsGetUByte(); + char magic2 = bsGetUByte(); + char magic3 = bsGetUByte(); + char magic4 = bsGetUByte(); + char magic5 = bsGetUByte(); + + if (magic0 == 0x17 && magic1 == 0x72 && magic2 == 0x45 + && magic3 == 0x38 && magic4 == 0x50 && magic5 == 0x90) + { + complete(); // end of file + } + else if (magic0 != 0x31 || + magic1 != 0x41 || + magic2 != 0x59 || + magic3 != 0x26 || + magic4 != 0x53 || + magic5 != 0x59) + { + this.currentState = CState.EOF; + var msg = String.Format("bad block header at offset 0x{0:X}", + this.input.Position); + throw new IOException(msg); + } + else + { + this.storedBlockCRC = bsGetInt(); + // Console.WriteLine(" stored block CRC : {0:X8}", this.storedBlockCRC); + + this.blockRandomised = (GetBits(1) == 1); + + // Lazily allocate data + if (this.data == null) + this.data = new DecompressionState(this.blockSize100k); + + // currBlockNo++; + getAndMoveToFrontDecode(); + + this.crc.Reset(); + this.currentState = CState.START_BLOCK; + } + } + + + private void EndBlock() + { + this.computedBlockCRC = (uint)this.crc.Crc32Result; + + // A bad CRC is considered a fatal error. + if (this.storedBlockCRC != this.computedBlockCRC) + { + // make next blocks readable without error + // (repair feature, not yet documented, not tested) + // this.computedCombinedCRC = (this.storedCombinedCRC << 1) + // | (this.storedCombinedCRC >> 31); + // this.computedCombinedCRC ^= this.storedBlockCRC; + + var msg = String.Format("BZip2 CRC error (expected {0:X8}, computed {1:X8})", + this.storedBlockCRC, this.computedBlockCRC); + throw new IOException(msg); + } + + // Console.WriteLine(" combined CRC (before): {0:X8}", this.computedCombinedCRC); + this.computedCombinedCRC = (this.computedCombinedCRC << 1) + | (this.computedCombinedCRC >> 31); + this.computedCombinedCRC ^= this.computedBlockCRC; + // Console.WriteLine(" computed block CRC : {0:X8}", this.computedBlockCRC); + // Console.WriteLine(" combined CRC (after) : {0:X8}", this.computedCombinedCRC); + // Console.WriteLine(); + } + + + private void complete() + { + this.storedCombinedCRC = bsGetInt(); + this.currentState = CState.EOF; + this.data = null; + + if (this.storedCombinedCRC != this.computedCombinedCRC) + { + var msg = String.Format("BZip2 CRC error (expected {0:X8}, computed {1:X8})", + this.storedCombinedCRC, this.computedCombinedCRC); + + throw new IOException(msg); + } + } + + /// + /// Close the stream. + /// + public override void Close() + { + Stream inShadow = this.input; + if (inShadow != null) + { + try + { + if (!this._leaveOpen) + inShadow.Close(); + } + finally + { + this.data = null; + this.input = null; + } + } + } + + + /// + /// Read n bits from input, right justifying the result. + /// + /// + /// + /// For example, if you read 1 bit, the result is either 0 + /// or 1. + /// + /// + /// + /// The number of bits to read, always between 1 and 32. + /// + private int GetBits(int n) + { + int bsLiveShadow = this.bsLive; + int bsBuffShadow = this.bsBuff; + + if (bsLiveShadow < n) + { + do + { + int thech = this.input.ReadByte(); + + if (thech < 0) + throw new IOException("unexpected end of stream"); + + // Console.WriteLine("R {0:X2}", thech); + + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + } while (bsLiveShadow < n); + + this.bsBuff = bsBuffShadow; + } + + this.bsLive = bsLiveShadow - n; + return (bsBuffShadow >> (bsLiveShadow - n)) & ((1 << n) - 1); + } + + + // private bool bsGetBit() + // { + // int bsLiveShadow = this.bsLive; + // int bsBuffShadow = this.bsBuff; + // + // if (bsLiveShadow < 1) + // { + // int thech = this.input.ReadByte(); + // + // if (thech < 0) + // { + // throw new IOException("unexpected end of stream"); + // } + // + // bsBuffShadow = (bsBuffShadow << 8) | thech; + // bsLiveShadow += 8; + // this.bsBuff = bsBuffShadow; + // } + // + // this.bsLive = bsLiveShadow - 1; + // return ((bsBuffShadow >> (bsLiveShadow - 1)) & 1) != 0; + // } + + private bool bsGetBit() + { + int bit = GetBits(1); + return bit != 0; + } + + private char bsGetUByte() + { + return (char)GetBits(8); + } + + private uint bsGetInt() + { + return (uint)((((((GetBits(8) << 8) | GetBits(8)) << 8) | GetBits(8)) << 8) | GetBits(8)); + } + + + /** + * Called by createHuffmanDecodingTables() exclusively. + */ + private static void hbCreateDecodeTables(int[] limit, + int[] bbase, int[] perm, char[] length, + int minLen, int maxLen, int alphaSize) + { + for (int i = minLen, pp = 0; i <= maxLen; i++) + { + for (int j = 0; j < alphaSize; j++) + { + if (length[j] == i) + { + perm[pp++] = j; + } + } + } + + for (int i = BZip2.MaxCodeLength; --i > 0;) + { + bbase[i] = 0; + limit[i] = 0; + } + + for (int i = 0; i < alphaSize; i++) + { + bbase[length[i] + 1]++; + } + + for (int i = 1, b = bbase[0]; i < BZip2.MaxCodeLength; i++) + { + b += bbase[i]; + bbase[i] = b; + } + + for (int i = minLen, vec = 0, b = bbase[i]; i <= maxLen; i++) + { + int nb = bbase[i + 1]; + vec += nb - b; + b = nb; + limit[i] = vec - 1; + vec <<= 1; + } + + for (int i = minLen + 1; i <= maxLen; i++) + { + bbase[i] = ((limit[i - 1] + 1) << 1) - bbase[i]; + } + } + + + + private void recvDecodingTables() + { + var s = this.data; + bool[] inUse = s.inUse; + byte[] pos = s.recvDecodingTables_pos; + //byte[] selector = s.selector; + + int inUse16 = 0; + + /* Receive the mapping table */ + for (int i = 0; i < 16; i++) + { + if (bsGetBit()) + { + inUse16 |= 1 << i; + } + } + + for (int i = 256; --i >= 0;) + { + inUse[i] = false; + } + + for (int i = 0; i < 16; i++) + { + if ((inUse16 & (1 << i)) != 0) + { + int i16 = i << 4; + for (int j = 0; j < 16; j++) + { + if (bsGetBit()) + { + inUse[i16 + j] = true; + } + } + } + } + + MakeMaps(); + int alphaSize = this.nInUse + 2; + + /* Now the selectors */ + int nGroups = GetBits(3); + int nSelectors = GetBits(15); + + for (int i = 0; i < nSelectors; i++) + { + int j = 0; + while (bsGetBit()) + { + j++; + } + s.selectorMtf[i] = (byte)j; + } + + /* Undo the MTF values for the selectors. */ + for (int v = nGroups; --v >= 0;) + { + pos[v] = (byte)v; + } + + for (int i = 0; i < nSelectors; i++) + { + int v = s.selectorMtf[i]; + byte tmp = pos[v]; + while (v > 0) + { + // nearly all times v is zero, 4 in most other cases + pos[v] = pos[v - 1]; + v--; + } + pos[0] = tmp; + s.selector[i] = tmp; + } + + char[][] len = s.temp_charArray2d; + + /* Now the coding tables */ + for (int t = 0; t < nGroups; t++) + { + int curr = GetBits(5); + char[] len_t = len[t]; + for (int i = 0; i < alphaSize; i++) + { + while (bsGetBit()) + { + curr += bsGetBit() ? -1 : 1; + } + len_t[i] = (char)curr; + } + } + + // finally create the Huffman tables + createHuffmanDecodingTables(alphaSize, nGroups); + } + + + /** + * Called by recvDecodingTables() exclusively. + */ + private void createHuffmanDecodingTables(int alphaSize, + int nGroups) + { + var s = this.data; + char[][] len = s.temp_charArray2d; + + for (int t = 0; t < nGroups; t++) + { + int minLen = 32; + int maxLen = 0; + char[] len_t = len[t]; + for (int i = alphaSize; --i >= 0;) + { + char lent = len_t[i]; + if (lent > maxLen) + maxLen = lent; + + if (lent < minLen) + minLen = lent; + } + hbCreateDecodeTables(s.gLimit[t], s.gBase[t], s.gPerm[t], len[t], minLen, + maxLen, alphaSize); + s.gMinlen[t] = minLen; + } + } + + + + private void getAndMoveToFrontDecode() + { + var s = this.data; + this.origPtr = GetBits(24); + + if (this.origPtr < 0) + throw new IOException("BZ_DATA_ERROR"); + if (this.origPtr > 10 + BZip2.BlockSizeMultiple * this.blockSize100k) + throw new IOException("BZ_DATA_ERROR"); + + recvDecodingTables(); + + byte[] yy = s.getAndMoveToFrontDecode_yy; + int limitLast = this.blockSize100k * BZip2.BlockSizeMultiple; + + /* + * Setting up the unzftab entries here is not strictly necessary, but it + * does save having to do it later in a separate pass, and so saves a + * block's worth of cache misses. + */ + for (int i = 256; --i >= 0;) + { + yy[i] = (byte)i; + s.unzftab[i] = 0; + } + + int groupNo = 0; + int groupPos = BZip2.G_SIZE - 1; + int eob = this.nInUse + 1; + int nextSym = getAndMoveToFrontDecode0(0); + int bsBuffShadow = this.bsBuff; + int bsLiveShadow = this.bsLive; + int lastShadow = -1; + int zt = s.selector[groupNo] & 0xff; + int[] base_zt = s.gBase[zt]; + int[] limit_zt = s.gLimit[zt]; + int[] perm_zt = s.gPerm[zt]; + int minLens_zt = s.gMinlen[zt]; + + while (nextSym != eob) + { + if ((nextSym == BZip2.RUNA) || (nextSym == BZip2.RUNB)) + { + int es = -1; + + for (int n = 1; true; n <<= 1) + { + if (nextSym == BZip2.RUNA) + { + es += n; + } + else if (nextSym == BZip2.RUNB) + { + es += n << 1; + } + else + { + break; + } + + if (groupPos == 0) + { + groupPos = BZip2.G_SIZE - 1; + zt = s.selector[++groupNo] & 0xff; + base_zt = s.gBase[zt]; + limit_zt = s.gLimit[zt]; + perm_zt = s.gPerm[zt]; + minLens_zt = s.gMinlen[zt]; + } + else + { + groupPos--; + } + + int zn = minLens_zt; + + // Inlined: + // int zvec = GetBits(zn); + while (bsLiveShadow < zn) + { + int thech = this.input.ReadByte(); + if (thech >= 0) + { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + else + { + throw new IOException("unexpected end of stream"); + } + } + int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) + & ((1 << zn) - 1); + bsLiveShadow -= zn; + + while (zvec > limit_zt[zn]) + { + zn++; + while (bsLiveShadow < 1) + { + int thech = this.input.ReadByte(); + if (thech >= 0) + { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + else + { + throw new IOException("unexpected end of stream"); + } + } + bsLiveShadow--; + zvec = (zvec << 1) + | ((bsBuffShadow >> bsLiveShadow) & 1); + } + nextSym = perm_zt[zvec - base_zt[zn]]; + } + + byte ch = s.seqToUnseq[yy[0]]; + s.unzftab[ch & 0xff] += es + 1; + + while (es-- >= 0) + { + s.ll8[++lastShadow] = ch; + } + + if (lastShadow >= limitLast) + throw new IOException("block overrun"); + } + else + { + if (++lastShadow >= limitLast) + throw new IOException("block overrun"); + + byte tmp = yy[nextSym - 1]; + s.unzftab[s.seqToUnseq[tmp] & 0xff]++; + s.ll8[lastShadow] = s.seqToUnseq[tmp]; + + /* + * This loop is hammered during decompression, hence avoid + * native method call overhead of System.Buffer.BlockCopy for very + * small ranges to copy. + */ + if (nextSym <= 16) + { + for (int j = nextSym - 1; j > 0;) + { + yy[j] = yy[--j]; + } + } + else + { + System.Buffer.BlockCopy(yy, 0, yy, 1, nextSym - 1); + } + + yy[0] = tmp; + + if (groupPos == 0) + { + groupPos = BZip2.G_SIZE - 1; + zt = s.selector[++groupNo] & 0xff; + base_zt = s.gBase[zt]; + limit_zt = s.gLimit[zt]; + perm_zt = s.gPerm[zt]; + minLens_zt = s.gMinlen[zt]; + } + else + { + groupPos--; + } + + int zn = minLens_zt; + + // Inlined: + // int zvec = GetBits(zn); + while (bsLiveShadow < zn) + { + int thech = this.input.ReadByte(); + if (thech >= 0) + { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + else + { + throw new IOException("unexpected end of stream"); + } + } + int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) + & ((1 << zn) - 1); + bsLiveShadow -= zn; + + while (zvec > limit_zt[zn]) + { + zn++; + while (bsLiveShadow < 1) + { + int thech = this.input.ReadByte(); + if (thech >= 0) + { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + else + { + throw new IOException("unexpected end of stream"); + } + } + bsLiveShadow--; + zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); + } + nextSym = perm_zt[zvec - base_zt[zn]]; + } + } + + this.last = lastShadow; + this.bsLive = bsLiveShadow; + this.bsBuff = bsBuffShadow; + } + + + private int getAndMoveToFrontDecode0(int groupNo) + { + var s = this.data; + int zt = s.selector[groupNo] & 0xff; + int[] limit_zt = s.gLimit[zt]; + int zn = s.gMinlen[zt]; + int zvec = GetBits(zn); + int bsLiveShadow = this.bsLive; + int bsBuffShadow = this.bsBuff; + + while (zvec > limit_zt[zn]) + { + zn++; + while (bsLiveShadow < 1) + { + int thech = this.input.ReadByte(); + + if (thech >= 0) + { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + else + { + throw new IOException("unexpected end of stream"); + } + } + bsLiveShadow--; + zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); + } + + this.bsLive = bsLiveShadow; + this.bsBuff = bsBuffShadow; + + return s.gPerm[zt][zvec - s.gBase[zt][zn]]; + } + + + private void SetupBlock() + { + if (this.data == null) + return; + + int i; + var s = this.data; + int[] tt = s.initTT(this.last + 1); + + // xxxx + + /* Check: unzftab entries in range. */ + for (i = 0; i <= 255; i++) + { + if (s.unzftab[i] < 0 || s.unzftab[i] > this.last) + throw new Exception("BZ_DATA_ERROR"); + } + + /* Actually generate cftab. */ + s.cftab[0] = 0; + for (i = 1; i <= 256; i++) s.cftab[i] = s.unzftab[i - 1]; + for (i = 1; i <= 256; i++) s.cftab[i] += s.cftab[i - 1]; + /* Check: cftab entries in range. */ + for (i = 0; i <= 256; i++) + { + if (s.cftab[i] < 0 || s.cftab[i] > this.last + 1) + { + var msg = String.Format("BZ_DATA_ERROR: cftab[{0}]={1} last={2}", + i, s.cftab[i], this.last); + throw new Exception(msg); + } + } + /* Check: cftab entries non-descending. */ + for (i = 1; i <= 256; i++) + { + if (s.cftab[i - 1] > s.cftab[i]) + throw new Exception("BZ_DATA_ERROR"); + } + + int lastShadow; + for (i = 0, lastShadow = this.last; i <= lastShadow; i++) + { + tt[s.cftab[s.ll8[i] & 0xff]++] = i; + } + + if ((this.origPtr < 0) || (this.origPtr >= tt.Length)) + throw new IOException("stream corrupted"); + + this.su_tPos = tt[this.origPtr]; + this.su_count = 0; + this.su_i2 = 0; + this.su_ch2 = 256; /* not a valid 8-bit byte value?, and not EOF */ + + if (this.blockRandomised) + { + this.su_rNToGo = 0; + this.su_rTPos = 0; + SetupRandPartA(); + } + else + { + SetupNoRandPartA(); + } + } + + + + private void SetupRandPartA() + { + if (this.su_i2 <= this.last) + { + this.su_chPrev = this.su_ch2; + int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff; + this.su_tPos = this.data.tt[this.su_tPos]; + if (this.su_rNToGo == 0) + { + this.su_rNToGo = Rand.Rnums(this.su_rTPos) - 1; + if (++this.su_rTPos == 512) + { + this.su_rTPos = 0; + } + } + else + { + this.su_rNToGo--; + } + this.su_ch2 = su_ch2Shadow ^= (this.su_rNToGo == 1) ? 1 : 0; + this.su_i2++; + this.currentChar = su_ch2Shadow; + this.currentState = CState.RAND_PART_B; + this.crc.UpdateCRC((byte)su_ch2Shadow); + } + else + { + EndBlock(); + InitBlock(); + SetupBlock(); + } + } + + private void SetupNoRandPartA() + { + if (this.su_i2 <= this.last) + { + this.su_chPrev = this.su_ch2; + int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff; + this.su_ch2 = su_ch2Shadow; + this.su_tPos = this.data.tt[this.su_tPos]; + this.su_i2++; + this.currentChar = su_ch2Shadow; + this.currentState = CState.NO_RAND_PART_B; + this.crc.UpdateCRC((byte)su_ch2Shadow); + } + else + { + this.currentState = CState.NO_RAND_PART_A; + EndBlock(); + InitBlock(); + SetupBlock(); + } + } + + private void SetupRandPartB() + { + if (this.su_ch2 != this.su_chPrev) + { + this.currentState = CState.RAND_PART_A; + this.su_count = 1; + SetupRandPartA(); + } + else if (++this.su_count >= 4) + { + this.su_z = (char)(this.data.ll8[this.su_tPos] & 0xff); + this.su_tPos = this.data.tt[this.su_tPos]; + if (this.su_rNToGo == 0) + { + this.su_rNToGo = Rand.Rnums(this.su_rTPos) - 1; + if (++this.su_rTPos == 512) + { + this.su_rTPos = 0; + } + } + else + { + this.su_rNToGo--; + } + this.su_j2 = 0; + this.currentState = CState.RAND_PART_C; + if (this.su_rNToGo == 1) + { + this.su_z ^= (char)1; + } + SetupRandPartC(); + } + else + { + this.currentState = CState.RAND_PART_A; + SetupRandPartA(); + } + } + + private void SetupRandPartC() + { + if (this.su_j2 < this.su_z) + { + this.currentChar = this.su_ch2; + this.crc.UpdateCRC((byte)this.su_ch2); + this.su_j2++; + } + else + { + this.currentState = CState.RAND_PART_A; + this.su_i2++; + this.su_count = 0; + SetupRandPartA(); + } + } + + private void SetupNoRandPartB() + { + if (this.su_ch2 != this.su_chPrev) + { + this.su_count = 1; + SetupNoRandPartA(); + } + else if (++this.su_count >= 4) + { + this.su_z = (char)(this.data.ll8[this.su_tPos] & 0xff); + this.su_tPos = this.data.tt[this.su_tPos]; + this.su_j2 = 0; + SetupNoRandPartC(); + } + else + { + SetupNoRandPartA(); + } + } + + private void SetupNoRandPartC() + { + if (this.su_j2 < this.su_z) + { + int su_ch2Shadow = this.su_ch2; + this.currentChar = su_ch2Shadow; + this.crc.UpdateCRC((byte)su_ch2Shadow); + this.su_j2++; + this.currentState = CState.NO_RAND_PART_C; + } + else + { + this.su_i2++; + this.su_count = 0; + SetupNoRandPartA(); + } + } + + private sealed class DecompressionState + { + // (with blockSize 900k) + readonly public bool[] inUse = new bool[256]; + readonly public byte[] seqToUnseq = new byte[256]; // 256 byte + readonly public byte[] selector = new byte[BZip2.MaxSelectors]; // 18002 byte + readonly public byte[] selectorMtf = new byte[BZip2.MaxSelectors]; // 18002 byte + + /** + * Freq table collected to save a pass over the data during + * decompression. + */ + public readonly int[] unzftab; + public readonly int[][] gLimit; + public readonly int[][] gBase; + public readonly int[][] gPerm; + public readonly int[] gMinlen; + + public readonly int[] cftab; + public readonly byte[] getAndMoveToFrontDecode_yy; + public readonly char[][] temp_charArray2d; + public readonly byte[] recvDecodingTables_pos; + // --------------- + // 60798 byte + + public int[] tt; // 3600000 byte + public byte[] ll8; // 900000 byte + + // --------------- + // 4560782 byte + // =============== + + public DecompressionState(int blockSize100k) + { + this.unzftab = new int[256]; // 1024 byte + + this.gLimit = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.gBase = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.gPerm = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.gMinlen = new int[BZip2.NGroups]; // 24 byte + + this.cftab = new int[257]; // 1028 byte + this.getAndMoveToFrontDecode_yy = new byte[256]; // 512 byte + this.temp_charArray2d = BZip2.InitRectangularArray(BZip2.NGroups, BZip2.MaxAlphaSize); + this.recvDecodingTables_pos = new byte[BZip2.NGroups]; // 6 byte + + this.ll8 = new byte[blockSize100k * BZip2.BlockSizeMultiple]; + } + + /** + * Initializes the tt array. + * + * This method is called when the required length of the array is known. + * I don't initialize it at construction time to avoid unneccessary + * memory allocation when compressing small files. + */ + public int[] initTT(int length) + { + int[] ttShadow = this.tt; + + // tt.length should always be >= length, but theoretically + // it can happen, if the compressor mixed small and large + // blocks. Normally only the last block will be smaller + // than others. + if ((ttShadow == null) || (ttShadow.Length < length)) + { + this.tt = ttShadow = new int[length]; + } + + return ttShadow; + } + } + + + } +} \ No newline at end of file diff --git a/SabreTools.Compression/BZip2/BZip2OutputStream.cs b/SabreTools.Compression/BZip2/BZip2OutputStream.cs new file mode 100644 index 0000000..6912715 --- /dev/null +++ b/SabreTools.Compression/BZip2/BZip2OutputStream.cs @@ -0,0 +1,530 @@ +//#define Trace + +// BZip2OutputStream.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-August-02 16:44:11> +// +// ------------------------------------------------------------------ +// +// This module defines the BZip2OutputStream class, which is a +// compressing stream that handles BZIP2. This code may have been +// derived in part from Apache commons source code. The license below +// applies to the original Apache code. +// +// ------------------------------------------------------------------ +// flymake: csc.exe /t:module BZip2InputStream.cs BZip2Compressor.cs Rand.cs BCRC32.cs @@FILE@@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +// Design Notes: +// +// This class follows the classic Decorator pattern: it is a Stream that +// wraps itself around a Stream, and in doing so provides bzip2 +// compression as callers Write into it. +// +// BZip2 is a straightforward data format: there are 4 magic bytes at +// the top of the file, followed by 1 or more compressed blocks. There +// is a small "magic byte" trailer after all compressed blocks. This +// class emits the magic bytes for the header and trailer, and relies on +// a BZip2Compressor to generate each of the compressed data blocks. +// +// BZip2 does byte-shredding - it uses partial fractions of bytes to +// represent independent pieces of information. This class relies on the +// BitWriter to adapt the bit-oriented BZip2 output to the byte-oriented +// model of the .NET Stream class. +// +// ---- +// +// Regarding the Apache code base: Most of the code in this particular +// class is related to stream operations, and is my own code. It largely +// does not rely on any code obtained from Apache commons. If you +// compare this code with the Apache commons BZip2OutputStream, you will +// see very little code that is common, except for the +// nearly-boilerplate structure that is common to all subtypes of +// System.IO.Stream. There may be some small remnants of code in this +// module derived from the Apache stuff, which is why I left the license +// in here. Most of the Apache commons compressor magic has been ported +// into the BZip2Compressor class. +// + +using System; +using System.IO; + +#nullable disable +namespace SabreTools.Compression.BZip2 +{ + /// + /// A write-only decorator stream that compresses data as it is + /// written using the BZip2 algorithm. + /// + public class BZip2OutputStream : System.IO.Stream + { + int totalBytesWrittenIn; + bool leaveOpen; + BZip2Compressor compressor; + uint combinedCRC; + Stream output; + BitWriter bw; + int blockSize100k; // 0...9 + + private TraceBits desiredTrace = TraceBits.Crc | TraceBits.Write; + + /// + /// Constructs a new BZip2OutputStream, that sends its + /// compressed output to the given output stream. + /// + /// + /// + /// The destination stream, to which compressed output will be sent. + /// + /// + /// + /// + /// This example reads a file, then compresses it with bzip2 file, + /// and writes the compressed data into a newly created file. + /// + /// + /// var fname = "logfile.log"; + /// using (var fs = File.OpenRead(fname)) + /// { + /// var outFname = fname + ".bz2"; + /// using (var output = File.Create(outFname)) + /// { + /// using (var compressor = new SabreTools.Compression.BZip2.BZip2OutputStream(output)) + /// { + /// byte[] buffer = new byte[2048]; + /// int n; + /// while ((n = fs.Read(buffer, 0, buffer.Length)) > 0) + /// { + /// compressor.Write(buffer, 0, n); + /// } + /// } + /// } + /// } + /// + /// + public BZip2OutputStream(Stream output) + : this(output, BZip2.MaxBlockSize, false) + { + } + + + /// + /// Constructs a new BZip2OutputStream with specified blocksize. + /// + /// the destination stream. + /// + /// The blockSize in units of 100000 bytes. + /// The valid range is 1..9. + /// + public BZip2OutputStream(Stream output, int blockSize) + : this(output, blockSize, false) + { + } + + + /// + /// Constructs a new BZip2OutputStream. + /// + /// the destination stream. + /// + /// whether to leave the captive stream open upon closing this stream. + /// + public BZip2OutputStream(Stream output, bool leaveOpen) + : this(output, BZip2.MaxBlockSize, leaveOpen) + { + } + + + /// + /// Constructs a new BZip2OutputStream with specified blocksize, + /// and explicitly specifies whether to leave the wrapped stream open. + /// + /// + /// the destination stream. + /// + /// The blockSize in units of 100000 bytes. + /// The valid range is 1..9. + /// + /// + /// whether to leave the captive stream open upon closing this stream. + /// + public BZip2OutputStream(Stream output, int blockSize, bool leaveOpen) + { + if (blockSize < BZip2.MinBlockSize || + blockSize > BZip2.MaxBlockSize) + { + var msg = String.Format("blockSize={0} is out of range; must be between {1} and {2}", + blockSize, + BZip2.MinBlockSize, BZip2.MaxBlockSize); + throw new ArgumentException(msg, "blockSize"); + } + + this.output = output; + if (!this.output.CanWrite) + throw new ArgumentException("The stream is not writable.", "output"); + + this.bw = new BitWriter(this.output); + this.blockSize100k = blockSize; + this.compressor = new BZip2Compressor(this.bw, blockSize); + this.leaveOpen = leaveOpen; + this.combinedCRC = 0; + EmitHeader(); + } + + + + + /// + /// Close the stream. + /// + /// + /// + /// This may or may not close the underlying stream. Check the + /// constructors that accept a bool value. + /// + /// + public override void Close() + { + if (output != null) + { + Stream o = this.output; + Finish(); + if (!leaveOpen) + o.Close(); + } + } + + + /// + /// Flush the stream. + /// + public override void Flush() + { + if (this.output != null) + { + this.bw.Flush(); + this.output.Flush(); + } + } + + private void EmitHeader() + { + var magic = new byte[] { + (byte) 'B', + (byte) 'Z', + (byte) 'h', + (byte) ('0' + this.blockSize100k) + }; + + // not necessary to shred the initial magic bytes + this.output.Write(magic, 0, magic.Length); + } + + private void EmitTrailer() + { + // A magic 48-bit number, 0x177245385090, to indicate the end + // of the last block. (sqrt(pi), if you want to know) + + TraceOutput(TraceBits.Write, "total written out: {0} (0x{0:X})", + this.bw.TotalBytesWrittenOut); + + // must shred + this.bw.WriteByte(0x17); + this.bw.WriteByte(0x72); + this.bw.WriteByte(0x45); + this.bw.WriteByte(0x38); + this.bw.WriteByte(0x50); + this.bw.WriteByte(0x90); + + this.bw.WriteInt(this.combinedCRC); + + this.bw.FinishAndPad(); + + TraceOutput(TraceBits.Write, "final total: {0} (0x{0:X})", + this.bw.TotalBytesWrittenOut); + } + + void Finish() + { + // Console.WriteLine("BZip2:Finish"); + + try + { + var totalBefore = this.bw.TotalBytesWrittenOut; + this.compressor.CompressAndWrite(); + TraceOutput(TraceBits.Write, "out block length (bytes): {0} (0x{0:X})", + this.bw.TotalBytesWrittenOut - totalBefore); + + TraceOutput(TraceBits.Crc, " combined CRC (before): {0:X8}", + this.combinedCRC); + this.combinedCRC = (this.combinedCRC << 1) | (this.combinedCRC >> 31); + this.combinedCRC ^= (uint)compressor.Crc32; + TraceOutput(TraceBits.Crc, " block CRC : {0:X8}", + this.compressor.Crc32); + TraceOutput(TraceBits.Crc, " combined CRC (final) : {0:X8}", + this.combinedCRC); + + EmitTrailer(); + } + finally + { + this.output = null; + this.compressor = null; + this.bw = null; + } + } + + + /// + /// The blocksize parameter specified at construction time. + /// + public int BlockSize + { + get { return this.blockSize100k; } + } + + + /// + /// Write data to the stream. + /// + /// + /// + /// + /// Use the BZip2OutputStream to compress data while writing: + /// create a BZip2OutputStream with a writable output stream. + /// Then call Write() on that BZip2OutputStream, providing + /// uncompressed data as input. The data sent to the output stream will + /// be the compressed form of the input data. + /// + /// + /// + /// A BZip2OutputStream can be used only for Write() not for Read(). + /// + /// + /// + /// + /// The buffer holding data to write to the stream. + /// the offset within that data array to find the first byte to write. + /// the number of bytes to write. + public override void Write(byte[] buffer, int offset, int count) + { + if (offset < 0) + throw new IndexOutOfRangeException(String.Format("offset ({0}) must be > 0", offset)); + if (count < 0) + throw new IndexOutOfRangeException(String.Format("count ({0}) must be > 0", count)); + if (offset + count > buffer.Length) + throw new IndexOutOfRangeException(String.Format("offset({0}) count({1}) bLength({2})", + offset, count, buffer.Length)); + if (this.output == null) + throw new IOException("the stream is not open"); + + if (count == 0) return; // nothing to do + + int bytesWritten = 0; + int bytesRemaining = count; + + do + { + int n = compressor.Fill(buffer, offset, bytesRemaining); + if (n != bytesRemaining) + { + // The compressor data block is full. Compress and + // write out the compressed data, then reset the + // compressor and continue. + + var totalBefore = this.bw.TotalBytesWrittenOut; + this.compressor.CompressAndWrite(); + TraceOutput(TraceBits.Write, "out block length (bytes): {0} (0x{0:X})", + this.bw.TotalBytesWrittenOut - totalBefore); + + // and now any remaining bits + TraceOutput(TraceBits.Write, + " remaining: {0} 0x{1:X}", + this.bw.NumRemainingBits, + this.bw.RemainingBits); + + TraceOutput(TraceBits.Crc, " combined CRC (before): {0:X8}", + this.combinedCRC); + this.combinedCRC = (this.combinedCRC << 1) | (this.combinedCRC >> 31); + this.combinedCRC ^= (uint)compressor.Crc32; + TraceOutput(TraceBits.Crc, " block CRC : {0:X8}", + compressor.Crc32); + TraceOutput(TraceBits.Crc, " combined CRC (after) : {0:X8}", + this.combinedCRC); + offset += n; + } + bytesRemaining -= n; + bytesWritten += n; + } while (bytesRemaining > 0); + + totalBytesWrittenIn += bytesWritten; + } + + + + + /// + /// Indicates whether the stream can be read. + /// + /// + /// The return value is always false. + /// + public override bool CanRead + { + get { return false; } + } + + /// + /// Indicates whether the stream supports Seek operations. + /// + /// + /// Always returns false. + /// + public override bool CanSeek + { + get { return false; } + } + + /// + /// Indicates whether the stream can be written. + /// + /// + /// The return value should always be true, unless and until the + /// object is disposed and closed. + /// + public override bool CanWrite + { + get + { + if (this.output == null) throw new ObjectDisposedException("BZip2Stream"); + return this.output.CanWrite; + } + } + + /// + /// Reading this property always throws a . + /// + public override long Length + { + get { throw new NotImplementedException(); } + } + + /// + /// The position of the stream pointer. + /// + /// + /// + /// Setting this property always throws a . Reading will return the + /// total number of uncompressed bytes written through. + /// + public override long Position + { + get + { + return this.totalBytesWrittenIn; + } + set { throw new NotImplementedException(); } + } + + /// + /// Calling this method always throws a . + /// + /// this is irrelevant, since it will always throw! + /// this is irrelevant, since it will always throw! + /// irrelevant! + public override long Seek(long offset, System.IO.SeekOrigin origin) + { + throw new NotImplementedException(); + } + + /// + /// Calling this method always throws a . + /// + /// this is irrelevant, since it will always throw! + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + /// + /// Calling this method always throws a . + /// + /// this parameter is never used + /// this parameter is never used + /// this parameter is never used + /// never returns anything; always throws + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + + // used only when Trace is defined + [Flags] + enum TraceBits : uint + { + None = 0, + Crc = 1, + Write = 2, + All = 0xffffffff, + } + + + [System.Diagnostics.ConditionalAttribute("Trace")] + private void TraceOutput(TraceBits bits, string format, params object[] varParams) + { + if ((bits & this.desiredTrace) != 0) + { + //lock(outputLock) + { + int tid = System.Threading.Thread.CurrentThread.GetHashCode(); +#if !SILVERLIGHT && !NETCF + Console.ForegroundColor = (ConsoleColor)(tid % 8 + 10); +#endif + Console.Write("{0:000} PBOS ", tid); + Console.WriteLine(format, varParams); +#if !SILVERLIGHT && !NETCF + Console.ResetColor(); +#endif + } + } + } + + + } + +} diff --git a/SabreTools.Compression/BZip2/BitWriter.cs b/SabreTools.Compression/BZip2/BitWriter.cs new file mode 100644 index 0000000..7455f66 --- /dev/null +++ b/SabreTools.Compression/BZip2/BitWriter.cs @@ -0,0 +1,246 @@ +// BitWriter.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-July-25 18:57:31> +// +// ------------------------------------------------------------------ +// +// This module defines the BitWriter class, which writes bits at a time +// to an output stream. It's used by the BZip2Compressor class, and by +// the BZip2OutputStream class and its parallel variant, +// ParallelBZip2OutputStream. +// +// ------------------------------------------------------------------ + +// +// Design notes: +// +// BZip2 employs byte-shredding in its data format - rather than +// aligning all data items in a compressed .bz2 file on byte barriers, +// the BZip2 format uses portions of bytes to represent independent +// pieces of information. This "shredding" starts with the first +// "randomised" bit - just 12 bytes or so into a bz2 file or stream. But +// the approach is used extensively in bzip2 files - sometimes 5 bits +// are used, sometimes 24 or 3 bits, sometimes just 1 bit, and so on. +// It's not possible to send this information directly to a stream in +// this form; Streams in .NET accept byte-oriented input. Therefore, +// when actually writing a bz2 file, the output data must be organized +// into a byte-aligned format before being written to the output stream. +// +// This BitWriter class provides the byte-shredding necessary for BZip2 +// output. Think of this class as an Adapter that enables Bit-oriented +// output to a standard byte-oriented .NET stream. This class writes +// data out to the captive output stream only after the data bits have +// been accumulated and aligned. For example, suppose that during +// operation, the BZip2 compressor emits 5 bits, then 24 bits, then 32 +// bits. When the first 5 bits are sent to the BitWriter, nothing is +// written to the output stream; instead these 5 bits are simply stored +// in the internal accumulator. When the next 24 bits are written, the +// first 3 bits are gathered with the accumulated bits. The resulting +// 5+3 constitutes an entire byte; the BitWriter then actually writes +// that byte to the output stream. This leaves 21 bits. BitWriter writes +// 2 more whole bytes (16 more bits), in 8-bit chunks, leaving 5 in the +// accumulator. BitWriter then follows the same procedure with the 32 +// new bits. And so on. +// +// A quick tour of the implementation: +// +// The accumulator is a uint - so it can accumulate at most 4 bytes of +// information. In practice because of the design of this class, it +// never accumulates more than 3 bytes. +// +// The Flush() method emits all whole bytes available. After calling +// Flush(), there may be between 0-7 bits yet to be emitted into the +// output stream. +// +// FinishAndPad() emits all data, including the last partial byte and +// any necessary padding. In effect, it establishes a byte-alignment +// barrier. To support bzip2, FinishAndPad() should be called only once +// for a bz2 file, after the last bit of data has been written through +// this adapter. Other binary file formats may use byte-alignment at +// various points within the file, and FinishAndPad() would support that +// scenario. +// +// The internal fn Reset() is used to reset the state of the adapter; +// this class is used by BZip2Compressor, instances of which get re-used +// by multiple distinct threads, for different blocks of data. +// + +using System; +using System.IO; + +namespace SabreTools.Compression.BZip2 +{ + internal class BitWriter + { + uint accumulator; + int nAccumulatedBits; + Stream output; + int totalBytesWrittenOut; + + public BitWriter(Stream s) + { + this.output = s; + } + + /// + /// Delivers the remaining bits, left-aligned, in a byte. + /// + /// + /// + /// This is valid only if NumRemainingBits is less than 8; + /// in other words it is valid only after a call to Flush(). + /// + /// + public byte RemainingBits + { + get + { + return (byte)(this.accumulator >> (32 - this.nAccumulatedBits) & 0xff); + } + } + + public int NumRemainingBits + { + get + { + return this.nAccumulatedBits; + } + } + + public int TotalBytesWrittenOut + { + get + { + return this.totalBytesWrittenOut; + } + } + + /// + /// Reset the BitWriter. + /// + /// + /// + /// This is useful when the BitWriter writes into a MemoryStream, and + /// is used by a BZip2Compressor, which itself is re-used for multiple + /// distinct data blocks. + /// + /// + public void Reset() + { + this.accumulator = 0; + this.nAccumulatedBits = 0; + this.totalBytesWrittenOut = 0; + this.output.Seek(0, SeekOrigin.Begin); + this.output.SetLength(0); + } + + /// + /// Write some number of bits from the given value, into the output. + /// + /// + /// + /// The nbits value should be a max of 25, for safety. For performance + /// reasons, this method does not check! + /// + /// + public void WriteBits(int nbits, uint value) + { + int nAccumulated = this.nAccumulatedBits; + uint u = this.accumulator; + + while (nAccumulated >= 8) + { + this.output.WriteByte((byte)(u >> 24 & 0xff)); + this.totalBytesWrittenOut++; + u <<= 8; + nAccumulated -= 8; + } + + this.accumulator = u | (value << (32 - nAccumulated - nbits)); + this.nAccumulatedBits = nAccumulated + nbits; + + // Console.WriteLine("WriteBits({0}, 0x{1:X2}) => {2:X8} n({3})", + // nbits, value, accumulator, nAccumulatedBits); + // Console.ReadLine(); + + // At this point the accumulator may contain up to 31 bits waiting for + // output. + } + + + /// + /// Write a full 8-bit byte into the output. + /// + public void WriteByte(byte b) + { + WriteBits(8, b); + } + + /// + /// Write four 8-bit bytes into the output. + /// + public void WriteInt(uint u) + { + WriteBits(8, (u >> 24) & 0xff); + WriteBits(8, (u >> 16) & 0xff); + WriteBits(8, (u >> 8) & 0xff); + WriteBits(8, u & 0xff); + } + + /// + /// Write all available byte-aligned bytes. + /// + /// + /// + /// This method writes no new output, but flushes any accumulated + /// bits. At completion, the accumulator may contain up to 7 + /// bits. + /// + /// + /// This is necessary when re-assembling output from N independent + /// compressors, one for each of N blocks. The output of any + /// particular compressor will in general have some fragment of a byte + /// remaining. This fragment needs to be accumulated into the + /// parent BZip2OutputStream. + /// + /// + public void Flush() + { + WriteBits(0, 0); + } + + + /// + /// Writes all available bytes, and emits padding for the final byte as + /// necessary. This must be the last method invoked on an instance of + /// BitWriter. + /// + public void FinishAndPad() + { + Flush(); + + if (this.NumRemainingBits > 0) + { + byte b = (byte)((this.accumulator >> 24) & 0xff); + this.output.WriteByte(b); + this.totalBytesWrittenOut++; + } + } + + } + +} \ No newline at end of file diff --git a/SabreTools.Compression/BZip2/CRC32.cs b/SabreTools.Compression/BZip2/CRC32.cs new file mode 100644 index 0000000..e2aa38b --- /dev/null +++ b/SabreTools.Compression/BZip2/CRC32.cs @@ -0,0 +1,815 @@ +// CRC32.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-August-02 18:25:54> +// +// ------------------------------------------------------------------ +// +// This module defines the CRC32 class, which can do the CRC32 algorithm, using +// arbitrary starting polynomials, and bit reversal. The bit reversal is what +// distinguishes this CRC-32 used in BZip2 from the CRC-32 that is used in PKZIP +// files, or GZIP files. This class does both. +// +// ------------------------------------------------------------------ + +using System; +using Interop = System.Runtime.InteropServices; + +#nullable disable +#pragma warning disable CS0618 +namespace SabreTools.Compression.BZip2 +{ + /// + /// Computes a CRC-32. The CRC-32 algorithm is parameterized - you + /// can set the polynomial and enable or disable bit + /// reversal. This can be used for GZIP, BZip2, or ZIP. + /// + /// + /// This type is used internally by DotNetZip; it is generally not used + /// directly by applications wishing to create, read, or manipulate zip + /// archive files. + /// + + [Interop.GuidAttribute("ebc25cf6-9120-4283-b972-0e5520d0000C")] + [Interop.ComVisible(true)] +#if !NETCF + [Interop.ClassInterface(Interop.ClassInterfaceType.AutoDispatch)] +#endif + public class CRC32 + { + /// + /// Indicates the total number of bytes applied to the CRC. + /// + public Int64 TotalBytesRead + { + get + { + return _TotalBytesRead; + } + } + + /// + /// Indicates the current CRC for all blocks slurped in. + /// + public Int32 Crc32Result + { + get + { + return unchecked((Int32)(~_register)); + } + } + + /// + /// Returns the CRC32 for the specified stream. + /// + /// The stream over which to calculate the CRC32 + /// the CRC32 calculation + public Int32 GetCrc32(System.IO.Stream input) + { + return GetCrc32AndCopy(input, null); + } + + /// + /// Returns the CRC32 for the specified stream, and writes the input into the + /// output stream. + /// + /// The stream over which to calculate the CRC32 + /// The stream into which to deflate the input + /// the CRC32 calculation + public Int32 GetCrc32AndCopy(System.IO.Stream input, System.IO.Stream output) + { + if (input == null) + throw new Exception("The input stream must not be null."); + + unchecked + { + byte[] buffer = new byte[BUFFER_SIZE]; + int readSize = BUFFER_SIZE; + + _TotalBytesRead = 0; + int count = input.Read(buffer, 0, readSize); + if (output != null) output.Write(buffer, 0, count); + _TotalBytesRead += count; + while (count > 0) + { + SlurpBlock(buffer, 0, count); + count = input.Read(buffer, 0, readSize); + if (output != null) output.Write(buffer, 0, count); + _TotalBytesRead += count; + } + + return (Int32)(~_register); + } + } + + + /// + /// Get the CRC32 for the given (word,byte) combo. This is a + /// computation defined by PKzip for PKZIP 2.0 (weak) encryption. + /// + /// The word to start with. + /// The byte to combine it with. + /// The CRC-ized result. + public Int32 ComputeCrc32(Int32 W, byte B) + { + return _InternalComputeCrc32((UInt32)W, B); + } + + internal Int32 _InternalComputeCrc32(UInt32 W, byte B) + { + return (Int32)(crc32Table[(W ^ B) & 0xFF] ^ (W >> 8)); + } + + + /// + /// Update the value for the running CRC32 using the given block of bytes. + /// This is useful when using the CRC32() class in a Stream. + /// + /// block of bytes to slurp + /// starting point in the block + /// how many bytes within the block to slurp + public void SlurpBlock(byte[] block, int offset, int count) + { + if (block == null) + throw new Exception("The data buffer must not be null."); + + // bzip algorithm + for (int i = 0; i < count; i++) + { + int x = offset + i; + byte b = block[x]; + if (this.reverseBits) + { + UInt32 temp = (_register >> 24) ^ b; + _register = (_register << 8) ^ crc32Table[temp]; + } + else + { + UInt32 temp = (_register & 0x000000FF) ^ b; + _register = (_register >> 8) ^ crc32Table[temp]; + } + } + _TotalBytesRead += count; + } + + + /// + /// Process one byte in the CRC. + /// + /// the byte to include into the CRC . + public void UpdateCRC(byte b) + { + if (this.reverseBits) + { + UInt32 temp = (_register >> 24) ^ b; + _register = (_register << 8) ^ crc32Table[temp]; + } + else + { + UInt32 temp = (_register & 0x000000FF) ^ b; + _register = (_register >> 8) ^ crc32Table[temp]; + } + } + + /// + /// Process a run of N identical bytes into the CRC. + /// + /// + /// + /// This method serves as an optimization for updating the CRC when a + /// run of identical bytes is found. Rather than passing in a buffer of + /// length n, containing all identical bytes b, this method accepts the + /// byte value and the length of the (virtual) buffer - the length of + /// the run. + /// + /// + /// the byte to include into the CRC. + /// the number of times that byte should be repeated. + public void UpdateCRC(byte b, int n) + { + while (n-- > 0) + { + if (this.reverseBits) + { + uint temp = (_register >> 24) ^ b; + _register = (_register << 8) ^ crc32Table[(temp >= 0) + ? temp + : (temp + 256)]; + } + else + { + UInt32 temp = (_register & 0x000000FF) ^ b; + _register = (_register >> 8) ^ crc32Table[(temp >= 0) + ? temp + : (temp + 256)]; + + } + } + } + + + + private static uint ReverseBits(uint data) + { + unchecked + { + uint ret = data; + ret = (ret & 0x55555555) << 1 | (ret >> 1) & 0x55555555; + ret = (ret & 0x33333333) << 2 | (ret >> 2) & 0x33333333; + ret = (ret & 0x0F0F0F0F) << 4 | (ret >> 4) & 0x0F0F0F0F; + ret = (ret << 24) | ((ret & 0xFF00) << 8) | ((ret >> 8) & 0xFF00) | (ret >> 24); + return ret; + } + } + + private static byte ReverseBits(byte data) + { + unchecked + { + uint u = (uint)data * 0x00020202; + uint m = 0x01044010; + uint s = u & m; + uint t = (u << 2) & (m << 1); + return (byte)((0x01001001 * (s + t)) >> 24); + } + } + + + + private void GenerateLookupTable() + { + crc32Table = new UInt32[256]; + unchecked + { + UInt32 dwCrc; + byte i = 0; + do + { + dwCrc = i; + for (byte j = 8; j > 0; j--) + { + if ((dwCrc & 1) == 1) + { + dwCrc = (dwCrc >> 1) ^ dwPolynomial; + } + else + { + dwCrc >>= 1; + } + } + if (reverseBits) + { + crc32Table[ReverseBits(i)] = ReverseBits(dwCrc); + } + else + { + crc32Table[i] = dwCrc; + } + i++; + } while (i!=0); + } + +#if VERBOSE + Console.WriteLine(); + Console.WriteLine("private static readonly UInt32[] crc32Table = {"); + for (int i = 0; i < crc32Table.Length; i+=4) + { + Console.Write(" "); + for (int j=0; j < 4; j++) + { + Console.Write(" 0x{0:X8}U,", crc32Table[i+j]); + } + Console.WriteLine(); + } + Console.WriteLine("};"); + Console.WriteLine(); +#endif + } + + + private uint gf2_matrix_times(uint[] matrix, uint vec) + { + uint sum = 0; + int i=0; + while (vec != 0) + { + if ((vec & 0x01)== 0x01) + sum ^= matrix[i]; + vec >>= 1; + i++; + } + return sum; + } + + private void gf2_matrix_square(uint[] square, uint[] mat) + { + for (int i = 0; i < 32; i++) + square[i] = gf2_matrix_times(mat, mat[i]); + } + + + + /// + /// Combines the given CRC32 value with the current running total. + /// + /// + /// This is useful when using a divide-and-conquer approach to + /// calculating a CRC. Multiple threads can each calculate a + /// CRC32 on a segment of the data, and then combine the + /// individual CRC32 values at the end. + /// + /// the crc value to be combined with this one + /// the length of data the CRC value was calculated on + public void Combine(int crc, int length) + { + uint[] even = new uint[32]; // even-power-of-two zeros operator + uint[] odd = new uint[32]; // odd-power-of-two zeros operator + + if (length == 0) + return; + + uint crc1= ~_register; + uint crc2= (uint) crc; + + // put operator for one zero bit in odd + odd[0] = this.dwPolynomial; // the CRC-32 polynomial + uint row = 1; + for (int i = 1; i < 32; i++) + { + odd[i] = row; + row <<= 1; + } + + // put operator for two zero bits in even + gf2_matrix_square(even, odd); + + // put operator for four zero bits in odd + gf2_matrix_square(odd, even); + + uint len2 = (uint) length; + + // apply len2 zeros to crc1 (first square will put the operator for one + // zero byte, eight zero bits, in even) + do { + // apply zeros operator for this bit of len2 + gf2_matrix_square(even, odd); + + if ((len2 & 1)== 1) + crc1 = gf2_matrix_times(even, crc1); + len2 >>= 1; + + if (len2 == 0) + break; + + // another iteration of the loop with odd and even swapped + gf2_matrix_square(odd, even); + if ((len2 & 1)==1) + crc1 = gf2_matrix_times(odd, crc1); + len2 >>= 1; + + + } while (len2 != 0); + + crc1 ^= crc2; + + _register= ~crc1; + + //return (int) crc1; + return; + } + + + /// + /// Create an instance of the CRC32 class using the default settings: no + /// bit reversal, and a polynomial of 0xEDB88320. + /// + public CRC32() : this(false) + { + } + + /// + /// Create an instance of the CRC32 class, specifying whether to reverse + /// data bits or not. + /// + /// + /// specify true if the instance should reverse data bits. + /// + /// + /// + /// In the CRC-32 used by BZip2, the bits are reversed. Therefore if you + /// want a CRC32 with compatibility with BZip2, you should pass true + /// here. In the CRC-32 used by GZIP and PKZIP, the bits are not + /// reversed; Therefore if you want a CRC32 with compatibility with + /// those, you should pass false. + /// + /// + public CRC32(bool reverseBits) : + this( unchecked((int)0xEDB88320), reverseBits) + { + } + + + /// + /// Create an instance of the CRC32 class, specifying the polynomial and + /// whether to reverse data bits or not. + /// + /// + /// The polynomial to use for the CRC, expressed in the reversed (LSB) + /// format: the highest ordered bit in the polynomial value is the + /// coefficient of the 0th power; the second-highest order bit is the + /// coefficient of the 1 power, and so on. Expressed this way, the + /// polynomial for the CRC-32C used in IEEE 802.3, is 0xEDB88320. + /// + /// + /// specify true if the instance should reverse data bits. + /// + /// + /// + /// + /// In the CRC-32 used by BZip2, the bits are reversed. Therefore if you + /// want a CRC32 with compatibility with BZip2, you should pass true + /// here for the reverseBits parameter. In the CRC-32 used by + /// GZIP and PKZIP, the bits are not reversed; Therefore if you want a + /// CRC32 with compatibility with those, you should pass false for the + /// reverseBits parameter. + /// + /// + public CRC32(int polynomial, bool reverseBits) + { + this.reverseBits = reverseBits; + this.dwPolynomial = (uint) polynomial; + this.GenerateLookupTable(); + } + + /// + /// Reset the CRC-32 class - clear the CRC "remainder register." + /// + /// + /// + /// Use this when employing a single instance of this class to compute + /// multiple, distinct CRCs on multiple, distinct data blocks. + /// + /// + public void Reset() + { + _register = 0xFFFFFFFFU; + } + + // private member vars + private UInt32 dwPolynomial; + private Int64 _TotalBytesRead; + private bool reverseBits; + private UInt32[] crc32Table; + private const int BUFFER_SIZE = 8192; + private UInt32 _register = 0xFFFFFFFFU; + } + + + /// + /// A Stream that calculates a CRC32 (a checksum) on all bytes read, + /// or on all bytes written. + /// + /// + /// + /// + /// This class can be used to verify the CRC of a ZipEntry when + /// reading from a stream, or to calculate a CRC when writing to a + /// stream. The stream should be used to either read, or write, but + /// not both. If you intermix reads and writes, the results are not + /// defined. + /// + /// + /// + /// This class is intended primarily for use internally by the + /// DotNetZip library. + /// + /// + public class CrcCalculatorStream : System.IO.Stream, System.IDisposable + { + private static readonly Int64 UnsetLengthLimit = -99; + + internal System.IO.Stream _innerStream; + private CRC32 _Crc32; + private Int64 _lengthLimit = -99; + private bool _leaveOpen; + + /// + /// The default constructor. + /// + /// + /// + /// Instances returned from this constructor will leave the underlying + /// stream open upon Close(). The stream uses the default CRC32 + /// algorithm, which implies a polynomial of 0xEDB88320. + /// + /// + /// The underlying stream + public CrcCalculatorStream(System.IO.Stream stream) + : this(true, CrcCalculatorStream.UnsetLengthLimit, stream, null) + { + } + + /// + /// The constructor allows the caller to specify how to handle the + /// underlying stream at close. + /// + /// + /// + /// The stream uses the default CRC32 algorithm, which implies a + /// polynomial of 0xEDB88320. + /// + /// + /// The underlying stream + /// true to leave the underlying stream + /// open upon close of the CrcCalculatorStream; false otherwise. + public CrcCalculatorStream(System.IO.Stream stream, bool leaveOpen) + : this(leaveOpen, CrcCalculatorStream.UnsetLengthLimit, stream, null) + { + } + + /// + /// A constructor allowing the specification of the length of the stream + /// to read. + /// + /// + /// + /// The stream uses the default CRC32 algorithm, which implies a + /// polynomial of 0xEDB88320. + /// + /// + /// Instances returned from this constructor will leave the underlying + /// stream open upon Close(). + /// + /// + /// The underlying stream + /// The length of the stream to slurp + public CrcCalculatorStream(System.IO.Stream stream, Int64 length) + : this(true, length, stream, null) + { + if (length < 0) + throw new ArgumentException("length"); + } + + /// + /// A constructor allowing the specification of the length of the stream + /// to read, as well as whether to keep the underlying stream open upon + /// Close(). + /// + /// + /// + /// The stream uses the default CRC32 algorithm, which implies a + /// polynomial of 0xEDB88320. + /// + /// + /// The underlying stream + /// The length of the stream to slurp + /// true to leave the underlying stream + /// open upon close of the CrcCalculatorStream; false otherwise. + public CrcCalculatorStream(System.IO.Stream stream, Int64 length, bool leaveOpen) + : this(leaveOpen, length, stream, null) + { + if (length < 0) + throw new ArgumentException("length"); + } + + /// + /// A constructor allowing the specification of the length of the stream + /// to read, as well as whether to keep the underlying stream open upon + /// Close(), and the CRC32 instance to use. + /// + /// + /// + /// The stream uses the specified CRC32 instance, which allows the + /// application to specify how the CRC gets calculated. + /// + /// + /// The underlying stream + /// The length of the stream to slurp + /// true to leave the underlying stream + /// open upon close of the CrcCalculatorStream; false otherwise. + /// the CRC32 instance to use to calculate the CRC32 + public CrcCalculatorStream(System.IO.Stream stream, Int64 length, bool leaveOpen, + CRC32 crc32) + : this(leaveOpen, length, stream, crc32) + { + if (length < 0) + throw new ArgumentException("length"); + } + + + // This ctor is private - no validation is done here. This is to allow the use + // of a (specific) negative value for the _lengthLimit, to indicate that there + // is no length set. So we validate the length limit in those ctors that use an + // explicit param, otherwise we don't validate, because it could be our special + // value. + private CrcCalculatorStream + (bool leaveOpen, Int64 length, System.IO.Stream stream, CRC32 crc32) + : base() + { + _innerStream = stream; + _Crc32 = crc32 ?? new CRC32(); + _lengthLimit = length; + _leaveOpen = leaveOpen; + } + + + /// + /// Gets the total number of bytes run through the CRC32 calculator. + /// + /// + /// + /// This is either the total number of bytes read, or the total number of + /// bytes written, depending on the direction of this stream. + /// + public Int64 TotalBytesSlurped + { + get { return _Crc32.TotalBytesRead; } + } + + /// + /// Provides the current CRC for all blocks slurped in. + /// + /// + /// + /// The running total of the CRC is kept as data is written or read + /// through the stream. read this property after all reads or writes to + /// get an accurate CRC for the entire stream. + /// + /// + public Int32 Crc + { + get { return _Crc32.Crc32Result; } + } + + /// + /// Indicates whether the underlying stream will be left open when the + /// CrcCalculatorStream is Closed. + /// + /// + /// + /// Set this at any point before calling . + /// + /// + public bool LeaveOpen + { + get { return _leaveOpen; } + set { _leaveOpen = value; } + } + + /// + /// Read from the stream + /// + /// the buffer to read + /// the offset at which to start + /// the number of bytes to read + /// the number of bytes actually read + public override int Read(byte[] buffer, int offset, int count) + { + int bytesToRead = count; + + // Need to limit the # of bytes returned, if the stream is intended to have + // a definite length. This is especially useful when returning a stream for + // the uncompressed data directly to the application. The app won't + // necessarily read only the UncompressedSize number of bytes. For example + // wrapping the stream returned from OpenReader() into a StreadReader() and + // calling ReadToEnd() on it, We can "over-read" the zip data and get a + // corrupt string. The length limits that, prevents that problem. + + if (_lengthLimit != CrcCalculatorStream.UnsetLengthLimit) + { + if (_Crc32.TotalBytesRead >= _lengthLimit) return 0; // EOF + Int64 bytesRemaining = _lengthLimit - _Crc32.TotalBytesRead; + if (bytesRemaining < count) bytesToRead = (int)bytesRemaining; + } + int n = _innerStream.Read(buffer, offset, bytesToRead); + if (n > 0) _Crc32.SlurpBlock(buffer, offset, n); + return n; + } + + /// + /// Write to the stream. + /// + /// the buffer from which to write + /// the offset at which to start writing + /// the number of bytes to write + public override void Write(byte[] buffer, int offset, int count) + { + if (count > 0) _Crc32.SlurpBlock(buffer, offset, count); + _innerStream.Write(buffer, offset, count); + } + + /// + /// Indicates whether the stream supports reading. + /// + public override bool CanRead + { + get { return _innerStream.CanRead; } + } + + /// + /// Indicates whether the stream supports seeking. + /// + /// + /// + /// Always returns false. + /// + /// + public override bool CanSeek + { + get { return false; } + } + + /// + /// Indicates whether the stream supports writing. + /// + public override bool CanWrite + { + get { return _innerStream.CanWrite; } + } + + /// + /// Flush the stream. + /// + public override void Flush() + { + _innerStream.Flush(); + } + + /// + /// Returns the length of the underlying stream. + /// + public override long Length + { + get + { + if (_lengthLimit == CrcCalculatorStream.UnsetLengthLimit) + return _innerStream.Length; + else return _lengthLimit; + } + } + + /// + /// The getter for this property returns the total bytes read. + /// If you use the setter, it will throw + /// . + /// + public override long Position + { + get { return _Crc32.TotalBytesRead; } + set { throw new NotSupportedException(); } + } + + /// + /// Seeking is not supported on this stream. This method always throws + /// + /// + /// N/A + /// N/A + /// N/A + public override long Seek(long offset, System.IO.SeekOrigin origin) + { + throw new NotSupportedException(); + } + + /// + /// This method always throws + /// + /// + /// N/A + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + + void IDisposable.Dispose() + { + Close(); + } + + /// + /// Closes the stream. + /// + public override void Close() + { + base.Close(); + if (!_leaveOpen) + _innerStream.Close(); + } + + } + +} \ No newline at end of file diff --git a/SabreTools.Compression/BZip2/Rand.cs b/SabreTools.Compression/BZip2/Rand.cs new file mode 100644 index 0000000..ba76d1b --- /dev/null +++ b/SabreTools.Compression/BZip2/Rand.cs @@ -0,0 +1,99 @@ +// Rand.cs +// ------------------------------------------------------------------ +// +// Copyright (c) 2011 Dino Chiesa. +// All rights reserved. +// +// This code module is part of DotNetZip, a zipfile class library. +// +// ------------------------------------------------------------------ +// +// This code is licensed under the Microsoft Public License. +// See the file License.txt for the license details. +// More info on: http://dotnetzip.codeplex.com +// +// ------------------------------------------------------------------ +// +// Last Saved: <2011-July-31 15:09:16> +// +// ------------------------------------------------------------------ +// +// This module defines a helper class for the BZip2 classes. This code +// is derived from the original BZip2 source code. +// +// ------------------------------------------------------------------ + + +namespace SabreTools.Compression.BZip2 +{ + internal static class Rand + { + private static int[] RNUMS = + { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 + }; + + + /// + /// Returns the "random" number at a specific index. + /// + /// the index + /// the random number + internal static int Rnums(int i) + { + return RNUMS[i]; + } + } + +} \ No newline at end of file