Update API, and more (#581)

Update API, and more:
 * remove "custom dictionary" support
 * c/encoder: fix #580: big-endian build
 * Java: reduce jar size
 * Java: speedup decoding
 * Java: add 32-bit CPU support
 * Java: make source code JS transpiler-ready
This commit is contained in:
Eugene Kliuchnikov
2017-08-04 10:02:56 +02:00
committed by GitHub
parent 0608253110
commit d63e8f75f5
65 changed files with 1061 additions and 2086 deletions

View File

@@ -42,12 +42,6 @@ java_test(
runtime_deps = [":test_lib"],
)
java_test(
name = "EnumTest",
test_class = "org.brotli.dec.EnumTest",
runtime_deps = [":test_lib"],
)
java_test(
name = "SynthTest",
test_class = "org.brotli.dec.SynthTest",

View File

@@ -6,52 +6,34 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
/**
* Bit reading helpers.
*/
final class BitReader {
/**
* Input byte buffer, consist of a ring-buffer and a "slack" region where bytes from the start of
* the ring-buffer are copied.
*/
private static final int CAPACITY = 1024;
private static final int SLACK = 16;
private static final int INT_BUFFER_SIZE = CAPACITY + SLACK;
private static final int BYTE_READ_SIZE = CAPACITY << 2;
private static final int BYTE_BUFFER_SIZE = INT_BUFFER_SIZE << 2;
// Possible values: {5, 6}. 5 corresponds to 32-bit build, 6 to 64-bit. This value is used for
// conditional compilation -> produced artifacts might be binary INCOMPATIBLE (JLS 13.2).
private static final int LOG_BITNESS = 6;
private static final int BITNESS = 1 << LOG_BITNESS;
private final byte[] byteBuffer = new byte[BYTE_BUFFER_SIZE];
private final int[] intBuffer = new int[INT_BUFFER_SIZE];
private final IntReader intReader = new IntReader();
private static final int BYTENESS = BITNESS / 8;
private static final int CAPACITY = 4096;
// After encountering the end of the input stream, this amount of zero bytes will be appended.
private static final int SLACK = 64;
private static final int BUFFER_SIZE = CAPACITY + SLACK;
// Don't bother to replenish the buffer while this number of bytes is available.
private static final int SAFEGUARD = 36;
private static final int WATERLINE = CAPACITY - SAFEGUARD;
private InputStream input;
// "Half" refers to "half of native integer type", i.e. on 64-bit machines it is 32-bit type,
// on 32-bit machines it is 16-bit.
private static final int HALF_BITNESS = BITNESS / 2;
private static final int HALF_SIZE = BYTENESS / 2;
private static final int HALVES_CAPACITY = CAPACITY / HALF_SIZE;
private static final int HALF_BUFFER_SIZE = BUFFER_SIZE / HALF_SIZE;
private static final int HALF_WATERLINE = WATERLINE / HALF_SIZE;
/**
* Input stream is finished.
*/
private boolean endOfStreamReached;
/**
* Pre-fetched bits.
*/
long accumulator;
/**
* Current bit-reading position in accumulator.
*/
int bitOffset;
/**
* Offset of next item in intBuffer.
*/
private int intOffset;
/* Number of bytes in unfinished "int" item. */
private int tailBytes = 0;
private static final int LOG_HALF_SIZE = LOG_BITNESS - 4;
/**
* Fills up the input buffer.
@@ -62,141 +44,148 @@ final class BitReader {
* buffer.
*/
// TODO: Split to check and read; move read outside of decoding loop.
static void readMoreInput(BitReader br) {
if (br.intOffset <= CAPACITY - 9) {
static void readMoreInput(State s) {
if (s.halfOffset <= HALF_WATERLINE) {
return;
}
if (br.endOfStreamReached) {
if (intAvailable(br) >= -2) {
if (s.endOfStreamReached != 0) {
if (halfAvailable(s) >= -2) {
return;
}
throw new BrotliRuntimeException("No more input");
}
int readOffset = br.intOffset << 2;
int bytesRead = BYTE_READ_SIZE - readOffset;
System.arraycopy(br.byteBuffer, readOffset, br.byteBuffer, 0, bytesRead);
br.intOffset = 0;
try {
while (bytesRead < BYTE_READ_SIZE) {
int len = br.input.read(br.byteBuffer, bytesRead, BYTE_READ_SIZE - bytesRead);
// EOF is -1 in Java, but 0 in C#.
if (len <= 0) {
br.endOfStreamReached = true;
br.tailBytes = bytesRead;
bytesRead += 3;
break;
}
bytesRead += len;
int readOffset = s.halfOffset << LOG_HALF_SIZE;
int bytesInBuffer = CAPACITY - readOffset;
// Move unused bytes to the head of the buffer.
Utils.copyBytesWithin(s.byteBuffer, 0, readOffset, CAPACITY);
s.halfOffset = 0;
while (bytesInBuffer < CAPACITY) {
int spaceLeft = CAPACITY - bytesInBuffer;
int len = Utils.readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
// EOF is -1 in Java, but 0 in C#.
if (len <= 0) {
s.endOfStreamReached = 1;
s.tailBytes = bytesInBuffer;
bytesInBuffer += HALF_SIZE - 1;
break;
}
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
bytesInBuffer += len;
}
IntReader.convert(br.intReader, bytesRead >> 2);
bytesToNibbles(s, bytesInBuffer);
}
static void checkHealth(BitReader br, boolean endOfStream) {
if (!br.endOfStreamReached) {
static void checkHealth(State s, int endOfStream) {
if (s.endOfStreamReached == 0) {
return;
}
int byteOffset = (br.intOffset << 2) + ((br.bitOffset + 7) >> 3) - 8;
if (byteOffset > br.tailBytes) {
int byteOffset = (s.halfOffset << LOG_HALF_SIZE) + ((s.bitOffset + 7) >> 3) - BYTENESS;
if (byteOffset > s.tailBytes) {
throw new BrotliRuntimeException("Read after end");
}
if (endOfStream && (byteOffset != br.tailBytes)) {
if ((endOfStream != 0) && (byteOffset != s.tailBytes)) {
throw new BrotliRuntimeException("Unused bytes after end");
}
}
/**
* Advances the Read buffer by 5 bytes to make room for reading next 24 bits.
*/
static void fillBitWindow(BitReader br) {
if (br.bitOffset >= 32) {
br.accumulator = ((long) br.intBuffer[br.intOffset++] << 32) | (br.accumulator >>> 32);
br.bitOffset -= 32;
static void fillBitWindow(State s) {
if (s.bitOffset >= HALF_BITNESS) {
if (BITNESS == 64) {
s.accumulator64 = ((long) s.intBuffer[s.halfOffset++] << HALF_BITNESS)
| (s.accumulator64 >>> HALF_BITNESS);
} else {
s.accumulator32 = ((int) s.shortBuffer[s.halfOffset++] << HALF_BITNESS)
| (s.accumulator32 >>> HALF_BITNESS);
}
s.bitOffset -= HALF_BITNESS;
}
}
/**
* Reads the specified number of bits from Read Buffer.
*/
static int readBits(BitReader br, int n) {
fillBitWindow(br);
int val = (int) (br.accumulator >>> br.bitOffset) & ((1 << n) - 1);
br.bitOffset += n;
static int peekBits(State s) {
if (BITNESS == 64) {
return (int) (s.accumulator64 >>> s.bitOffset);
} else {
return s.accumulator32 >>> s.bitOffset;
}
}
static int readFewBits(State s, int n) {
fillBitWindow(s);
int val = peekBits(s) & ((1 << n) - 1);
s.bitOffset += n;
return val;
}
/**
* Initialize bit reader.
*
* <p> Initialisation turns bit reader to a ready state. Also a number of bytes is prefetched to
* accumulator. Because of that this method may block until enough data could be read from input.
*
* @param br BitReader POJO
* @param input data source
*/
static void init(BitReader br, InputStream input) {
if (br.input != null) {
throw new IllegalStateException("Bit reader already has associated input stream");
}
IntReader.init(br.intReader, br.byteBuffer, br.intBuffer);
br.input = input;
br.accumulator = 0;
br.bitOffset = 64;
br.intOffset = CAPACITY;
br.endOfStreamReached = false;
prepare(br);
}
private static void prepare(BitReader br) {
readMoreInput(br);
checkHealth(br, false);
fillBitWindow(br);
fillBitWindow(br);
}
static void reload(BitReader br) {
if (br.bitOffset == 64) {
prepare(br);
static int readBits(State s, int n) {
if (HALF_BITNESS >= 24) {
return readFewBits(s, n);
} else {
if (n <= 16) {
return readFewBits(s, n);
} else if (s.bitOffset + n <= BITNESS) {
return readFewBits(s, n);
} else {
int lo = readFewBits(s, 16);
int hi = readFewBits(s, n - 16);
return lo | (hi << 16);
}
}
}
static void close(BitReader br) throws IOException {
InputStream is = br.input;
br.input = null;
if (is != null) {
is.close();
static void initBitReader(State s) {
s.byteBuffer = new byte[BUFFER_SIZE];
if (BITNESS == 64) {
s.accumulator64 = 0;
s.intBuffer = new int[HALF_BUFFER_SIZE];
} else {
s.accumulator32 = 0;
s.shortBuffer = new short[HALF_BUFFER_SIZE];
}
s.bitOffset = BITNESS;
s.halfOffset = HALVES_CAPACITY;
s.endOfStreamReached = 0;
prepare(s);
}
static void prepare(State s) {
readMoreInput(s);
checkHealth(s, 0);
fillBitWindow(s);
fillBitWindow(s);
}
static void reload(State s) {
if (s.bitOffset == BITNESS) {
prepare(s);
}
}
static void jumpToByteBoundary(BitReader br) {
int padding = (64 - br.bitOffset) & 7;
static void jumpToByteBoundary(State s) {
int padding = (BITNESS - s.bitOffset) & 7;
if (padding != 0) {
int paddingBits = BitReader.readBits(br, padding);
int paddingBits = readBits(s, padding);
if (paddingBits != 0) {
throw new BrotliRuntimeException("Corrupted padding bits");
}
}
}
static int intAvailable(BitReader br) {
int limit = CAPACITY;
if (br.endOfStreamReached) {
limit = (br.tailBytes + 3) >> 2;
static int halfAvailable(State s) {
int limit = HALVES_CAPACITY;
if (s.endOfStreamReached != 0) {
limit = (s.tailBytes + (HALF_SIZE - 1)) >> LOG_HALF_SIZE;
}
return limit - br.intOffset;
return limit - s.halfOffset;
}
static void copyBytes(BitReader br, byte[] data, int offset, int length) {
if ((br.bitOffset & 7) != 0) {
static void copyBytes(State s, byte[] data, int offset, int length) {
if ((s.bitOffset & 7) != 0) {
throw new BrotliRuntimeException("Unaligned copyBytes");
}
// Drain accumulator.
while ((br.bitOffset != 64) && (length != 0)) {
data[offset++] = (byte) (br.accumulator >>> br.bitOffset);
br.bitOffset += 8;
while ((s.bitOffset != BITNESS) && (length != 0)) {
data[offset++] = (byte) peekBits(s);
s.bitOffset += 8;
length--;
}
if (length == 0) {
@@ -204,43 +193,63 @@ final class BitReader {
}
// Get data from shadow buffer with "sizeof(int)" granularity.
int copyInts = Math.min(intAvailable(br), length >> 2);
if (copyInts > 0) {
int readOffset = br.intOffset << 2;
System.arraycopy(br.byteBuffer, readOffset, data, offset, copyInts << 2);
offset += copyInts << 2;
length -= copyInts << 2;
br.intOffset += copyInts;
int copyNibbles = Math.min(halfAvailable(s), length >> LOG_HALF_SIZE);
if (copyNibbles > 0) {
int readOffset = s.halfOffset << LOG_HALF_SIZE;
int delta = copyNibbles << LOG_HALF_SIZE;
System.arraycopy(s.byteBuffer, readOffset, data, offset, delta);
offset += delta;
length -= delta;
s.halfOffset += copyNibbles;
}
if (length == 0) {
return;
}
// Read tail bytes.
if (intAvailable(br) > 0) {
if (halfAvailable(s) > 0) {
// length = 1..3
fillBitWindow(br);
fillBitWindow(s);
while (length != 0) {
data[offset++] = (byte) (br.accumulator >>> br.bitOffset);
br.bitOffset += 8;
data[offset++] = (byte) peekBits(s);
s.bitOffset += 8;
length--;
}
checkHealth(br, false);
checkHealth(s, 0);
return;
}
// Now it is possible to copy bytes directly.
try {
while (length > 0) {
int len = br.input.read(data, offset, length);
if (len == -1) {
throw new BrotliRuntimeException("Unexpected end of input");
}
offset += len;
length -= len;
while (length > 0) {
int len = Utils.readInput(s.input, data, offset, length);
if (len == -1) {
throw new BrotliRuntimeException("Unexpected end of input");
}
offset += len;
length -= len;
}
}
/**
* Translates bytes to halves (int/short).
*/
static void bytesToNibbles(State s, int byteLen) {
byte[] byteBuffer = s.byteBuffer;
int halfLen = byteLen >> LOG_HALF_SIZE;
if (BITNESS == 64) {
int[] intBuffer = s.intBuffer;
for (int i = 0; i < halfLen; ++i) {
intBuffer[i] = ((byteBuffer[i * 4] & 0xFF))
| ((byteBuffer[(i * 4) + 1] & 0xFF) << 8)
| ((byteBuffer[(i * 4) + 2] & 0xFF) << 16)
| ((byteBuffer[(i * 4) + 3] & 0xFF) << 24);
}
} else {
short[] shortBuffer = s.shortBuffer;
for (int i = 0; i < halfLen; ++i) {
shortBuffer[i] = (short) ((byteBuffer[i * 2] & 0xFF)
| ((byteBuffer[(i * 2) + 1] & 0xFF) << 8));
}
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
}
}

View File

@@ -21,11 +21,11 @@ public class BitReaderTest {
@Test
public void testReadAfterEos() {
BitReader reader = new BitReader();
BitReader.init(reader, new ByteArrayInputStream(new byte[1]));
State reader = new State();
Decode.initState(reader, new ByteArrayInputStream(new byte[1]));
BitReader.readBits(reader, 9);
try {
BitReader.checkHealth(reader, false);
BitReader.checkHealth(reader, 0);
} catch (BrotliRuntimeException ex) {
// This exception is expected.
return;

View File

@@ -50,7 +50,7 @@ public class BrotliInputStream extends InputStream {
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source) throws IOException {
this(source, DEFAULT_INTERNAL_BUFFER_SIZE, null);
this(source, DEFAULT_INTERNAL_BUFFER_SIZE);
}
/**
@@ -67,25 +67,6 @@ public class BrotliInputStream extends InputStream {
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source, int byteReadBufferSize) throws IOException {
this(source, byteReadBufferSize, null);
}
/**
* Creates a {@link InputStream} wrapper that decompresses brotli data.
*
* <p> For byte-by-byte reading ({@link #read()}) internal buffer of specified size is
* allocated and used.
*
* <p> Will block the thread until first kilobyte of data of source is available.
*
* @param source compressed data source
* @param byteReadBufferSize size of internal buffer used in case of
* byte-by-byte reading
* @param customDictionary custom dictionary data; {@code null} if not used
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source, int byteReadBufferSize,
byte[] customDictionary) throws IOException {
if (byteReadBufferSize <= 0) {
throw new IllegalArgumentException("Bad buffer size:" + byteReadBufferSize);
} else if (source == null) {
@@ -95,13 +76,10 @@ public class BrotliInputStream extends InputStream {
this.remainingBufferBytes = 0;
this.bufferOffset = 0;
try {
State.setInput(state, source);
Decode.initState(state, source);
} catch (BrotliRuntimeException ex) {
throw new IOException("Brotli decoder initialization failed", ex);
}
if (customDictionary != null) {
Decode.setCustomDictionary(state, customDictionary);
}
}
/**
@@ -109,7 +87,7 @@ public class BrotliInputStream extends InputStream {
*/
@Override
public void close() throws IOException {
State.close(state);
Decode.close(state);
}
/**

View File

@@ -11,152 +11,48 @@ package org.brotli.dec;
*/
final class Context {
static final int[] LOOKUP = {
// CONTEXT_UTF8, last byte.
// ASCII range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
static final int[] LOOKUP = new int[2048];
// UTF8 continuation byte range.
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
private static final String UTF_MAP = " !! ! \"#$##%#$&'##(#)#+++++++++"
+ "+((&*'##,---,---,-----,-----,-----&#'###.///.///./////./////./////&#'# ";
private static final String UTF_RLE = "A/* ': & : $ \u0081 @";
// UTF8 lead byte range.
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
private static void unpackLookupTable(int[] lookup, String map, String rle) {
// LSB6, MSB6, SIGNED
for (int i = 0; i < 256; ++i) {
lookup[i] = i & 0x3F;
lookup[512 + i] = i >> 2;
lookup[1792 + i] = 2 + (i >> 6);
}
// UTF8
for (int i = 0; i < 128; ++i) {
lookup[1024 + i] = 4 * (map.charAt(i) - 32);
}
for (int i = 0; i < 64; ++i) {
lookup[1152 + i] = i & 1;
lookup[1216 + i] = 2 + (i & 1);
}
int offset = 1280;
for (int k = 0; k < 19; ++k) {
int value = k & 3;
int rep = rle.charAt(k) - 32;
for (int i = 0; i < rep; ++i) {
lookup[offset++] = value;
}
}
// SIGNED
for (int i = 0; i < 16; ++i) {
lookup[1792 + i] = 1;
lookup[2032 + i] = 6;
}
lookup[1792] = 0;
lookup[2047] = 7;
for (int i = 0; i < 256; ++i) {
lookup[1536 + i] = lookup[1792 + i] << 3;
}
}
// CONTEXT_UTF8 second last byte.
// ASCII range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
// UTF8 continuation byte range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// UTF8 lead byte range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// CONTEXT_SIGNED, second last byte.
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
// CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits.
0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56,
// CONTEXT_LSB6, last byte.
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
// CONTEXT_MSB6, last byte.
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15,
16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23,
24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27,
28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31,
32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35,
36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39,
40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43,
44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47,
48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51,
52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55,
56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59,
60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63,
// CONTEXT_{M,L}SB6, second last byte,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
static final int[] LOOKUP_OFFSETS = {
// CONTEXT_LSB6
1024, 1536,
// CONTEXT_MSB6
1280, 1536,
// CONTEXT_UTF8
0, 256,
// CONTEXT_SIGNED
768, 512
};
static {
unpackLookupTable(LOOKUP, UTF_MAP, UTF_RLE);
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -21,6 +21,14 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class DecodeTest {
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
}
return result;
}
private byte[] decompress(byte[] data, boolean byByte) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
@@ -49,35 +57,9 @@ public class DecodeTest {
return output.toByteArray();
}
private byte[] decompressWithDictionary(byte[] data, byte[] dictionary) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
ByteArrayOutputStream output = new ByteArrayOutputStream();
BrotliInputStream brotliInput = new BrotliInputStream(
input, BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, dictionary);
while (true) {
int len = brotliInput.read(buffer, 0, buffer.length);
if (len <= 0) {
break;
}
output.write(buffer, 0, len);
}
brotliInput.close();
return output.toByteArray();
}
private void checkDecodeResourceWithDictionary(String expected, String compressed,
String dictionary) throws IOException {
byte[] expectedBytes = Transform.readUniBytes(expected);
byte[] compressedBytes = Transform.readUniBytes(compressed);
byte[] dictionaryBytes = Transform.readUniBytes(dictionary);
byte[] actual = decompressWithDictionary(compressedBytes, dictionaryBytes);
assertArrayEquals(expectedBytes, actual);
}
private void checkDecodeResource(String expected, String compressed) throws IOException {
byte[] expectedBytes = Transform.readUniBytes(expected);
byte[] compressedBytes = Transform.readUniBytes(compressed);
byte[] expectedBytes = readUniBytes(expected);
byte[] compressedBytes = readUniBytes(compressed);
byte[] actual = decompress(compressedBytes, false);
assertArrayEquals(expectedBytes, actual);
byte[] actualByByte = decompress(compressedBytes, true);
@@ -168,20 +150,11 @@ public class DecodeTest {
+ "\u0091\u00FF\u0087\u00E9\u001E");
}
@Test
public void testFoxFox() throws IOException {
checkDecodeResourceWithDictionary(
"The quick brown fox jumps over the lazy dog",
"\u001B*\u0000\u0000 \u0000\u00C2\u0098\u00B0\u00CA\u0001",
"The quick brown fox jumps over the lazy dog");
}
@Test
public void testUtils() {
new Context();
new Decode();
new Dictionary();
new Huffman();
new Prefix();
}
}

View File

@@ -48,19 +48,4 @@ public final class Dictionary {
/* Might have been set when {@link DictionaryData} was loaded.*/
return data;
}
static final int[] OFFSETS_BY_LENGTH = {
0, 0, 0, 0, 0, 4096, 9216, 21504, 35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280, 122016
};
static final int[] SIZE_BITS_BY_LENGTH = {
0, 0, 0, 0, 10, 10, 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 7, 7, 8, 7, 7, 6, 6, 5, 5
};
static final int MIN_WORD_LENGTH = 4;
static final int MAX_WORD_LENGTH = 24;
static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + MAX_WORD_LENGTH + 8;
}

File diff suppressed because one or more lines are too long

View File

@@ -1,53 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.TreeSet;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for Enum-like classes.
*/
@RunWith(JUnit4.class)
public class EnumTest {
private void checkEnumClass(Class<?> clazz) {
TreeSet<Integer> values = new TreeSet<Integer>();
for (Field f : clazz.getDeclaredFields()) {
assertEquals("int", f.getType().getName());
assertEquals(Modifier.FINAL | Modifier.STATIC, f.getModifiers());
Integer value = null;
try {
value = f.getInt(null);
} catch (IllegalAccessException ex) {
fail("Inaccessible field");
}
assertFalse(values.contains(value));
values.add(value);
}
assertEquals(0, values.first().intValue());
assertEquals(values.size(), values.last() + 1);
}
@Test
public void testRunningState() {
checkEnumClass(RunningState.class);
}
@Test
public void testWordTransformType() {
checkEnumClass(WordTransformType.class);
}
}

View File

@@ -11,12 +11,6 @@ package org.brotli.dec;
*/
final class Huffman {
/**
* Maximum possible Huffman table size for an alphabet size of 704, max code length 15 and root
* table bits 8.
*/
static final int HUFFMAN_MAX_TABLE_SIZE = 1080;
private static final int MAX_LENGTH = 15;
/**

View File

@@ -1,57 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Contains a collection of huffman trees with the same alphabet size.
*/
final class HuffmanTreeGroup {
/**
* The maximal alphabet size in this group.
*/
private int alphabetSize;
/**
* Storage for Huffman lookup tables.
*/
int[] codes;
/**
* Offsets of distinct lookup tables in {@link #codes} storage.
*/
int[] trees;
/**
* Initializes the Huffman tree group.
*
* @param group POJO to be initialised
* @param alphabetSize the maximal alphabet size in this group
* @param n number of Huffman codes
*/
static void init(HuffmanTreeGroup group, int alphabetSize, int n) {
group.alphabetSize = alphabetSize;
group.codes = new int[n * Huffman.HUFFMAN_MAX_TABLE_SIZE];
group.trees = new int[n];
}
/**
* Decodes Huffman trees from input stream and constructs lookup tables.
*
* @param group target POJO
* @param br data source
*/
static void decode(HuffmanTreeGroup group, BitReader br) {
int next = 0;
int n = group.trees.length;
for (int i = 0; i < n; i++) {
group.trees[i] = next;
Decode.readHuffmanCode(group.alphabetSize, group.codes, next, br);
next += Huffman.HUFFMAN_MAX_TABLE_SIZE;
}
}
}

View File

@@ -1,36 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Byte-to-int conversion magic.
*/
final class IntReader {
private byte[] byteBuffer;
private int[] intBuffer;
static void init(IntReader ir, byte[] byteBuffer, int[] intBuffer) {
ir.byteBuffer = byteBuffer;
ir.intBuffer = intBuffer;
}
/**
* Translates bytes to ints.
*
* NB: intLen == 4 * byteSize!
* NB: intLen should be less or equal to intBuffer length.
*/
static void convert(IntReader ir, int intLen) {
for (int i = 0; i < intLen; ++i) {
ir.intBuffer[i] = ((ir.byteBuffer[i * 4] & 0xFF))
| ((ir.byteBuffer[(i * 4) + 1] & 0xFF) << 8)
| ((ir.byteBuffer[(i * 4) + 2] & 0xFF) << 16)
| ((ir.byteBuffer[(i * 4) + 3] & 0xFF) << 24);
}
}
}

View File

@@ -1,53 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Lookup tables to map prefix codes to value ranges.
*
* <p> This is used during decoding of the block lengths, literal insertion lengths and copy
* lengths.
*
* <p> Range represents values: [offset, offset + 2 ^ n_bits)
*/
final class Prefix {
static final int[] BLOCK_LENGTH_OFFSET = {
1, 5, 9, 13, 17, 25, 33, 41, 49, 65, 81, 97, 113, 145, 177, 209, 241, 305, 369, 497,
753, 1265, 2289, 4337, 8433, 16625
};
static final int[] BLOCK_LENGTH_N_BITS = {
2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24
};
static final int[] INSERT_LENGTH_OFFSET = {
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210,
22594
};
static final int[] INSERT_LENGTH_N_BITS = {
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24
};
static final int[] COPY_LENGTH_OFFSET = {
2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094,
2118
};
static final int[] COPY_LENGTH_N_BITS = {
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24
};
static final int[] INSERT_RANGE_LUT = {
0, 0, 8, 8, 0, 16, 8, 16, 16
};
static final int[] COPY_RANGE_LUT = {
0, 8, 0, 8, 16, 0, 16, 8, 16
};
}

View File

@@ -1,26 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Enumeration of decoding state-machine.
*/
final class RunningState {
static final int UNINITIALIZED = 0;
static final int BLOCK_START = 1;
static final int COMPRESSED_BLOCK_START = 2;
static final int MAIN_LOOP = 3;
static final int READ_METADATA = 4;
static final int COPY_UNCOMPRESSED = 5;
static final int INSERT_LOOP = 6;
static final int COPY_LOOP = 7;
static final int COPY_WRAP_BUFFER = 8;
static final int TRANSFORM = 9;
static final int FINISHED = 10;
static final int CLOSED = 11;
static final int WRITE = 12;
}

View File

@@ -6,51 +6,58 @@
package org.brotli.dec;
import static org.brotli.dec.RunningState.BLOCK_START;
import static org.brotli.dec.RunningState.CLOSED;
import static org.brotli.dec.RunningState.UNINITIALIZED;
import java.io.IOException;
import java.io.InputStream;
final class State {
int runningState = UNINITIALIZED;
int nextRunningState;
final BitReader br = new BitReader();
byte[] ringBuffer;
final int[] blockTypeTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE];
final int[] blockLenTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE];
byte[] contextModes;
byte[] contextMap;
byte[] distContextMap;
byte[] output;
byte[] byteBuffer; // BitReader
// Current meta-block header information.
short[] shortBuffer; // BitReader
int[] intBuffer; // BitReader
int[] rings;
int[] blockTrees;
int[] hGroup0;
int[] hGroup1;
int[] hGroup2;
long accumulator64; // BitReader: pre-fetched bits.
int runningState; // Default value is 0 == Decode.UNINITIALIZED
int nextRunningState;
int accumulator32; // BitReader: pre-fetched bits.
int bitOffset; // BitReader: bit-reading position in accumulator.
int halfOffset; // BitReader: offset of next item in intBuffer/shortBuffer.
int tailBytes; // BitReader: number of bytes in unfinished half.
int endOfStreamReached; // BitReader: input stream is finished.
int metaBlockLength;
boolean inputEnd;
boolean isUncompressed;
boolean isMetadata;
final HuffmanTreeGroup hGroup0 = new HuffmanTreeGroup();
final HuffmanTreeGroup hGroup1 = new HuffmanTreeGroup();
final HuffmanTreeGroup hGroup2 = new HuffmanTreeGroup();
final int[] blockLength = new int[3];
final int[] numBlockTypes = new int[3];
final int[] blockTypeRb = new int[6];
final int[] distRb = {16, 15, 11, 4};
int pos = 0;
int maxDistance = 0;
int distRbIdx = 0;
boolean trivialLiteralContext = false;
int literalTreeIndex = 0;
int inputEnd;
int isUncompressed;
int isMetadata;
int literalBlockLength;
int numLiteralBlockTypes;
int commandBlockLength;
int numCommandBlockTypes;
int distanceBlockLength;
int numDistanceBlockTypes;
int pos;
int maxDistance;
int distRbIdx;
int trivialLiteralContext;
int literalTreeIndex;
int literalTree;
int j;
int insertLength;
byte[] contextModes;
byte[] contextMap;
int contextMapSlice;
int distContextMapSlice;
int contextLookupOffset1;
int contextLookupOffset2;
int treeCommandOffset;
int distanceCode;
byte[] distContextMap;
int numDirectDistanceCodes;
int distancePostfixMask;
int distancePostfixBits;
@@ -59,62 +66,23 @@ final class State {
int copyDst;
int maxBackwardDistance;
int maxRingBufferSize;
int ringBufferSize = 0;
long expectedTotalSize = 0;
byte[] customDictionary = new byte[0];
int bytesToIgnore = 0;
int ringBufferSize;
int expectedTotalSize;
int bytesToIgnore;
int outputOffset;
int outputLength;
int outputUsed;
int bytesWritten;
int bytesToWrite;
byte[] output;
// TODO: Update to current spec.
private static int decodeWindowBits(BitReader br) {
if (BitReader.readBits(br, 1) == 0) {
return 16;
}
int n = BitReader.readBits(br, 3);
if (n != 0) {
return 17 + n;
}
n = BitReader.readBits(br, 3);
if (n != 0) {
return 8 + n;
}
return 17;
}
InputStream input; // BitReader
/**
* Associate input with decoder state.
*
* @param state uninitialized state without associated input
* @param input compressed data source
*/
static void setInput(State state, InputStream input) {
if (state.runningState != UNINITIALIZED) {
throw new IllegalStateException("State MUST be uninitialized");
}
BitReader.init(state.br, input);
int windowBits = decodeWindowBits(state.br);
if (windowBits == 9) { /* Reserved case for future expansion. */
throw new BrotliRuntimeException("Invalid 'windowBits' code");
}
state.maxRingBufferSize = 1 << windowBits;
state.maxBackwardDistance = state.maxRingBufferSize - 16;
state.runningState = BLOCK_START;
}
static void close(State state) throws IOException {
if (state.runningState == UNINITIALIZED) {
throw new IllegalStateException("State MUST be initialized");
}
if (state.runningState == CLOSED) {
return;
}
state.runningState = CLOSED;
BitReader.close(state.br);
State() {
this.ringBuffer = new byte[0];
this.rings = new int[10];
this.rings[0] = 16;
this.rings[1] = 15;
this.rings[2] = 11;
this.rings[3] = 4;
}
}

View File

@@ -22,6 +22,14 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class SynthTest {
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
}
return result;
}
private byte[] decompress(byte[] data) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
@@ -40,7 +48,7 @@ public class SynthTest {
private void checkSynth(byte[] compressed, boolean expectSuccess,
String expectedOutput) {
byte[] expected = Transform.readUniBytes(expectedOutput);
byte[] expected = readUniBytes(expectedOutput);
try {
byte[] actual = decompress(compressed);
if (!expectSuccess) {

View File

@@ -6,27 +6,6 @@
package org.brotli.dec;
import static org.brotli.dec.WordTransformType.IDENTITY;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_1;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_2;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_3;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_4;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_5;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_6;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_7;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_9;
import static org.brotli.dec.WordTransformType.OMIT_LAST_1;
import static org.brotli.dec.WordTransformType.OMIT_LAST_2;
import static org.brotli.dec.WordTransformType.OMIT_LAST_3;
import static org.brotli.dec.WordTransformType.OMIT_LAST_4;
import static org.brotli.dec.WordTransformType.OMIT_LAST_5;
import static org.brotli.dec.WordTransformType.OMIT_LAST_6;
import static org.brotli.dec.WordTransformType.OMIT_LAST_7;
import static org.brotli.dec.WordTransformType.OMIT_LAST_8;
import static org.brotli.dec.WordTransformType.OMIT_LAST_9;
import static org.brotli.dec.WordTransformType.UPPERCASE_ALL;
import static org.brotli.dec.WordTransformType.UPPERCASE_FIRST;
import java.nio.ByteBuffer;
/**
@@ -34,185 +13,82 @@ import java.nio.ByteBuffer;
*/
final class Transform {
private final byte[] prefix;
private final int type;
private final byte[] suffix;
static final int NUM_TRANSFORMS = 121;
private static final int[] TRANSFORMS = new int[NUM_TRANSFORMS * 3];
private static final byte[] PREFIX_SUFFIX = new byte[217];
private static final int[] PREFIX_SUFFIX_HEADS = new int[51];
Transform(String prefix, int type, String suffix) {
this.prefix = readUniBytes(prefix);
this.type = type;
this.suffix = readUniBytes(suffix);
}
// Bundle of 0-terminated strings.
private static final String PREFIX_SUFFIX_SRC = "# #s #, #e #.# the #.com/#\u00C2\u00A0# of # and"
+ " # in # to #\"#\">#\n#]# for # a # that #. # with #'# from # by #. The # on # as # is #ing"
+ " #\n\t#:#ed #(# at #ly #=\"# of the #. This #,# not #er #al #='#ful #ive #less #est #ize #"
+ "ous #";
private static final String TRANSFORMS_SRC = " !! ! , *! &! \" ! ) * * - ! # ! #!*! "
+ "+ ,$ ! - % . / # 0 1 . \" 2 3!* 4% ! # / 5 6 7 8 0 1 & $ 9 + : "
+ " ; < ' != > ?! 4 @ 4 2 & A *# ( B C& ) % ) !*# *-% A +! *. D! %' & E *6 F "
+ " G% ! *A *% H! D I!+! J!+ K +- *4! A L!*4 M N +6 O!*% +.! K *G P +%( ! G *D +D "
+ " Q +# *K!*G!+D!+# +G +A +4!+% +K!+4!*D!+K!*K";
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
private static void unpackTransforms(byte[] prefixSuffix, int[] prefixSuffixHeads,
int[] transforms, String prefixSuffixSrc, String transformsSrc) {
int n = prefixSuffixSrc.length();
int index = 1;
for (int i = 0; i < n; ++i) {
char c = prefixSuffixSrc.charAt(i);
prefixSuffix[i] = (byte) c;
if (c == 35) { // == #
prefixSuffixHeads[index++] = i + 1;
prefixSuffix[i] = 0;
}
}
for (int i = 0; i < NUM_TRANSFORMS * 3; ++i) {
transforms[i] = transformsSrc.charAt(i) - 32;
}
return result;
}
static final Transform[] TRANSFORMS = {
new Transform("", IDENTITY, ""),
new Transform("", IDENTITY, " "),
new Transform(" ", IDENTITY, " "),
new Transform("", OMIT_FIRST_1, ""),
new Transform("", UPPERCASE_FIRST, " "),
new Transform("", IDENTITY, " the "),
new Transform(" ", IDENTITY, ""),
new Transform("s ", IDENTITY, " "),
new Transform("", IDENTITY, " of "),
new Transform("", UPPERCASE_FIRST, ""),
new Transform("", IDENTITY, " and "),
new Transform("", OMIT_FIRST_2, ""),
new Transform("", OMIT_LAST_1, ""),
new Transform(", ", IDENTITY, " "),
new Transform("", IDENTITY, ", "),
new Transform(" ", UPPERCASE_FIRST, " "),
new Transform("", IDENTITY, " in "),
new Transform("", IDENTITY, " to "),
new Transform("e ", IDENTITY, " "),
new Transform("", IDENTITY, "\""),
new Transform("", IDENTITY, "."),
new Transform("", IDENTITY, "\">"),
new Transform("", IDENTITY, "\n"),
new Transform("", OMIT_LAST_3, ""),
new Transform("", IDENTITY, "]"),
new Transform("", IDENTITY, " for "),
new Transform("", OMIT_FIRST_3, ""),
new Transform("", OMIT_LAST_2, ""),
new Transform("", IDENTITY, " a "),
new Transform("", IDENTITY, " that "),
new Transform(" ", UPPERCASE_FIRST, ""),
new Transform("", IDENTITY, ". "),
new Transform(".", IDENTITY, ""),
new Transform(" ", IDENTITY, ", "),
new Transform("", OMIT_FIRST_4, ""),
new Transform("", IDENTITY, " with "),
new Transform("", IDENTITY, "'"),
new Transform("", IDENTITY, " from "),
new Transform("", IDENTITY, " by "),
new Transform("", OMIT_FIRST_5, ""),
new Transform("", OMIT_FIRST_6, ""),
new Transform(" the ", IDENTITY, ""),
new Transform("", OMIT_LAST_4, ""),
new Transform("", IDENTITY, ". The "),
new Transform("", UPPERCASE_ALL, ""),
new Transform("", IDENTITY, " on "),
new Transform("", IDENTITY, " as "),
new Transform("", IDENTITY, " is "),
new Transform("", OMIT_LAST_7, ""),
new Transform("", OMIT_LAST_1, "ing "),
new Transform("", IDENTITY, "\n\t"),
new Transform("", IDENTITY, ":"),
new Transform(" ", IDENTITY, ". "),
new Transform("", IDENTITY, "ed "),
new Transform("", OMIT_FIRST_9, ""),
new Transform("", OMIT_FIRST_7, ""),
new Transform("", OMIT_LAST_6, ""),
new Transform("", IDENTITY, "("),
new Transform("", UPPERCASE_FIRST, ", "),
new Transform("", OMIT_LAST_8, ""),
new Transform("", IDENTITY, " at "),
new Transform("", IDENTITY, "ly "),
new Transform(" the ", IDENTITY, " of "),
new Transform("", OMIT_LAST_5, ""),
new Transform("", OMIT_LAST_9, ""),
new Transform(" ", UPPERCASE_FIRST, ", "),
new Transform("", UPPERCASE_FIRST, "\""),
new Transform(".", IDENTITY, "("),
new Transform("", UPPERCASE_ALL, " "),
new Transform("", UPPERCASE_FIRST, "\">"),
new Transform("", IDENTITY, "=\""),
new Transform(" ", IDENTITY, "."),
new Transform(".com/", IDENTITY, ""),
new Transform(" the ", IDENTITY, " of the "),
new Transform("", UPPERCASE_FIRST, "'"),
new Transform("", IDENTITY, ". This "),
new Transform("", IDENTITY, ","),
new Transform(".", IDENTITY, " "),
new Transform("", UPPERCASE_FIRST, "("),
new Transform("", UPPERCASE_FIRST, "."),
new Transform("", IDENTITY, " not "),
new Transform(" ", IDENTITY, "=\""),
new Transform("", IDENTITY, "er "),
new Transform(" ", UPPERCASE_ALL, " "),
new Transform("", IDENTITY, "al "),
new Transform(" ", UPPERCASE_ALL, ""),
new Transform("", IDENTITY, "='"),
new Transform("", UPPERCASE_ALL, "\""),
new Transform("", UPPERCASE_FIRST, ". "),
new Transform(" ", IDENTITY, "("),
new Transform("", IDENTITY, "ful "),
new Transform(" ", UPPERCASE_FIRST, ". "),
new Transform("", IDENTITY, "ive "),
new Transform("", IDENTITY, "less "),
new Transform("", UPPERCASE_ALL, "'"),
new Transform("", IDENTITY, "est "),
new Transform(" ", UPPERCASE_FIRST, "."),
new Transform("", UPPERCASE_ALL, "\">"),
new Transform(" ", IDENTITY, "='"),
new Transform("", UPPERCASE_FIRST, ","),
new Transform("", IDENTITY, "ize "),
new Transform("", UPPERCASE_ALL, "."),
new Transform("\u00c2\u00a0", IDENTITY, ""),
new Transform(" ", IDENTITY, ","),
new Transform("", UPPERCASE_FIRST, "=\""),
new Transform("", UPPERCASE_ALL, "=\""),
new Transform("", IDENTITY, "ous "),
new Transform("", UPPERCASE_ALL, ", "),
new Transform("", UPPERCASE_FIRST, "='"),
new Transform(" ", UPPERCASE_FIRST, ","),
new Transform(" ", UPPERCASE_ALL, "=\""),
new Transform(" ", UPPERCASE_ALL, ", "),
new Transform("", UPPERCASE_ALL, ","),
new Transform("", UPPERCASE_ALL, "("),
new Transform("", UPPERCASE_ALL, ". "),
new Transform(" ", UPPERCASE_ALL, "."),
new Transform("", UPPERCASE_ALL, "='"),
new Transform(" ", UPPERCASE_ALL, ". "),
new Transform(" ", UPPERCASE_FIRST, "=\""),
new Transform(" ", UPPERCASE_ALL, "='"),
new Transform(" ", UPPERCASE_FIRST, "='")
};
static {
unpackTransforms(PREFIX_SUFFIX, PREFIX_SUFFIX_HEADS, TRANSFORMS, PREFIX_SUFFIX_SRC,
TRANSFORMS_SRC);
}
static int transformDictionaryWord(byte[] dst, int dstOffset, ByteBuffer data, int wordOffset,
int len, Transform transform) {
int len, int transformIndex) {
int offset = dstOffset;
int transformOffset = 3 * transformIndex;
int transformPrefix = PREFIX_SUFFIX_HEADS[TRANSFORMS[transformOffset]];
int transformType = TRANSFORMS[transformOffset + 1];
int transformSuffix = PREFIX_SUFFIX_HEADS[TRANSFORMS[transformOffset + 2]];
// Copy prefix.
byte[] string = transform.prefix;
int tmp = string.length;
int i = 0;
// In most cases tmp < 10 -> no benefits from System.arrayCopy
while (i < tmp) {
dst[offset++] = string[i++];
while (PREFIX_SUFFIX[transformPrefix] != 0) {
dst[offset++] = PREFIX_SUFFIX[transformPrefix++];
}
// Copy trimmed word.
int op = transform.type;
tmp = WordTransformType.getOmitFirst(op);
if (tmp > len) {
tmp = len;
int omitFirst = transformType >= 12 ? (transformType - 11) : 0;
if (omitFirst > len) {
omitFirst = len;
}
wordOffset += tmp;
len -= tmp;
len -= WordTransformType.getOmitLast(op);
i = len;
wordOffset += omitFirst;
len -= omitFirst;
len -= transformType <= 9 ? transformType : 0; // Omit last.
int i = len;
while (i > 0) {
dst[offset++] = data.get(wordOffset++);
i--;
}
if (op == UPPERCASE_ALL || op == UPPERCASE_FIRST) {
// Ferment.
if (transformType == 11 || transformType == 10) {
int uppercaseOffset = offset - len;
if (op == UPPERCASE_FIRST) {
if (transformType == 10) {
len = 1;
}
while (len > 0) {
tmp = dst[uppercaseOffset] & 0xFF;
int tmp = dst[uppercaseOffset] & 0xFF;
if (tmp < 0xc0) {
if (tmp >= 'a' && tmp <= 'z') {
if (tmp >= 97 && tmp <= 122) { // in [a..z] range
dst[uppercaseOffset] ^= (byte) 32;
}
uppercaseOffset += 1;
@@ -230,11 +106,8 @@ final class Transform {
}
// Copy suffix.
string = transform.suffix;
tmp = string.length;
i = 0;
while (i < tmp) {
dst[offset++] = string[i++];
while (PREFIX_SUFFIX[transformSuffix] != 0) {
dst[offset++] = PREFIX_SUFFIX[transformSuffix++];
}
return offset - dstOffset;

View File

@@ -34,23 +34,21 @@ public class TransformTest {
@Test
public void testTrimAll() {
byte[] output = new byte[2];
byte[] output = new byte[0];
byte[] input = {119, 111, 114, 100}; // "word"
Transform transform = new Transform("[", WordTransformType.OMIT_FIRST_5, "]");
Transform.transformDictionaryWord(
output, 0, ByteBuffer.wrap(input), 0, input.length, transform);
byte[] expectedOutput = {91, 93}; // "[]"
output, 0, ByteBuffer.wrap(input), 0, input.length, 39);
byte[] expectedOutput = new byte[0];
assertArrayEquals(expectedOutput, output);
}
@Test
public void testCapitalize() {
byte[] output = new byte[8];
byte[] output = new byte[6];
byte[] input = {113, -61, -90, -32, -92, -86}; // "qæप"
Transform transform = new Transform("[", WordTransformType.UPPERCASE_ALL, "]");
Transform.transformDictionaryWord(
output, 0, ByteBuffer.wrap(input), 0, input.length, transform);
byte[] expectedOutput = {91, 81, -61, -122, -32, -92, -81, 93}; // "[QÆय]"
output, 0, ByteBuffer.wrap(input), 0, input.length, 44);
byte[] expectedOutput = {81, -61, -122, -32, -92, -81}; // "QÆय"
assertArrayEquals(expectedOutput, output);
}
@@ -62,9 +60,9 @@ public class TransformTest {
byte[] testWord = {111, 49, 50, 51, 52, 53, 54, 55, 56, 57, 97, 98, 99, 100, 101, 102};
byte[] output = new byte[2259];
int offset = 0;
for (int i = 0; i < Transform.TRANSFORMS.length; ++i) {
for (int i = 0; i < Transform.NUM_TRANSFORMS; ++i) {
offset += Transform.transformDictionaryWord(
output, offset, ByteBuffer.wrap(testWord), 0, testWord.length, Transform.TRANSFORMS[i]);
output, offset, ByteBuffer.wrap(testWord), 0, testWord.length, i);
output[offset++] = -1;
}
assertEquals(output.length, offset);

View File

@@ -6,6 +6,9 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
/**
* A set of utility methods.
*/
@@ -25,11 +28,11 @@ final class Utils {
* @param offset the first byte to fill
* @param length number of bytes to change
*/
static void fillWithZeroes(byte[] dest, int offset, int length) {
int cursor = 0;
while (cursor < length) {
int step = Math.min(cursor + 1024, length) - cursor;
System.arraycopy(BYTE_ZEROES, 0, dest, offset + cursor, step);
static void fillBytesWithZeroes(byte[] dest, int start, int end) {
int cursor = start;
while (cursor < end) {
int step = Math.min(cursor + 1024, end) - cursor;
System.arraycopy(BYTE_ZEROES, 0, dest, cursor, step);
cursor += step;
}
}
@@ -44,12 +47,28 @@ final class Utils {
* @param offset the first item to fill
* @param length number of item to change
*/
static void fillWithZeroes(int[] dest, int offset, int length) {
int cursor = 0;
while (cursor < length) {
int step = Math.min(cursor + 1024, length) - cursor;
System.arraycopy(INT_ZEROES, 0, dest, offset + cursor, step);
static void fillIntsWithZeroes(int[] dest, int start, int end) {
int cursor = start;
while (cursor < end) {
int step = Math.min(cursor + 1024, end) - cursor;
System.arraycopy(INT_ZEROES, 0, dest, cursor, step);
cursor += step;
}
}
static void copyBytesWithin(byte[] bytes, int target, int start, int end) {
System.arraycopy(bytes, start, bytes, target, end - start);
}
static int readInput(InputStream src, byte[] dst, int offset, int length) {
try {
return src.read(dst, offset, length);
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
}
static void closeInput(InputStream src) throws IOException {
src.close();
}
}

View File

@@ -1,45 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Enumeration of all possible word transformations.
*
* <p>There are two simple types of transforms: omit X first/last symbols, two character-case
* transforms and the identity transform.
*/
final class WordTransformType {
static final int IDENTITY = 0;
static final int OMIT_LAST_1 = 1;
static final int OMIT_LAST_2 = 2;
static final int OMIT_LAST_3 = 3;
static final int OMIT_LAST_4 = 4;
static final int OMIT_LAST_5 = 5;
static final int OMIT_LAST_6 = 6;
static final int OMIT_LAST_7 = 7;
static final int OMIT_LAST_8 = 8;
static final int OMIT_LAST_9 = 9;
static final int UPPERCASE_FIRST = 10;
static final int UPPERCASE_ALL = 11;
static final int OMIT_FIRST_1 = 12;
static final int OMIT_FIRST_2 = 13;
static final int OMIT_FIRST_3 = 14;
static final int OMIT_FIRST_4 = 15;
static final int OMIT_FIRST_5 = 16;
static final int OMIT_FIRST_6 = 17;
static final int OMIT_FIRST_7 = 18;
static final int OMIT_FIRST_8 = 19;
static final int OMIT_FIRST_9 = 20;
static int getOmitFirst(int type) {
return type >= OMIT_FIRST_1 ? (type - OMIT_FIRST_1 + 1) : 0;
}
static int getOmitLast(int type) {
return type <= OMIT_LAST_9 ? (type - OMIT_LAST_1 + 1) : 0;
}
}

View File

@@ -27,13 +27,8 @@ public class BrotliDecoderChannel extends Decoder implements ReadableByteChannel
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize,
ByteBuffer customDictionary) throws IOException {
super(source, bufferSize, customDictionary);
}
public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize) throws IOException {
super(source, bufferSize, null);
super(source, bufferSize);
}
public BrotliDecoderChannel(ReadableByteChannel source) throws IOException {

View File

@@ -8,7 +8,6 @@ package org.brotli.wrapper.dec;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
/**
@@ -25,15 +24,10 @@ public class BrotliInputStream extends InputStream {
*
* @param source underlying source
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliInputStream(InputStream source, int bufferSize, ByteBuffer customDictionary)
public BrotliInputStream(InputStream source, int bufferSize)
throws IOException {
this.decoder = new Decoder(Channels.newChannel(source), bufferSize, customDictionary);
}
public BrotliInputStream(InputStream source, int bufferSize) throws IOException {
this.decoder = new Decoder(Channels.newChannel(source), bufferSize, null);
this.decoder = new Decoder(Channels.newChannel(source), bufferSize);
}
public BrotliInputStream(InputStream source) throws IOException {

View File

@@ -14,7 +14,7 @@ import java.util.ArrayList;
/**
* Base class for InputStream / Channel implementations.
*/
class Decoder {
public class Decoder {
private final ReadableByteChannel source;
private final DecoderJNI.Wrapper decoder;
ByteBuffer buffer;
@@ -26,7 +26,7 @@ class Decoder {
* @param source underlying source
* @param inputBufferSize read buffer size
*/
public Decoder(ReadableByteChannel source, int inputBufferSize, ByteBuffer customDictionary)
public Decoder(ReadableByteChannel source, int inputBufferSize)
throws IOException {
if (inputBufferSize <= 0) {
throw new IllegalArgumentException("buffer size must be positive");
@@ -35,7 +35,7 @@ class Decoder {
throw new NullPointerException("source can not be null");
}
this.source = source;
this.decoder = new DecoderJNI.Wrapper(inputBufferSize, customDictionary);
this.decoder = new DecoderJNI.Wrapper(inputBufferSize);
}
private void fail(String message) throws IOException {
@@ -119,7 +119,7 @@ class Decoder {
* Decodes the given data buffer.
*/
public static byte[] decompress(byte[] data) throws IOException {
DecoderJNI.Wrapper decoder = new DecoderJNI.Wrapper(data.length, null);
DecoderJNI.Wrapper decoder = new DecoderJNI.Wrapper(data.length);
ArrayList<byte[]> output = new ArrayList<byte[]>();
int totalOutputSize = 0;
try {

View File

@@ -13,7 +13,7 @@ import java.nio.ByteBuffer;
* JNI wrapper for brotli decoder.
*/
class DecoderJNI {
private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary);
private static native ByteBuffer nativeCreate(long[] context);
private static native void nativePush(long[] context, int length);
private static native ByteBuffer nativePull(long[] context);
private static native void nativeDestroy(long[] context);
@@ -31,12 +31,9 @@ class DecoderJNI {
private final ByteBuffer inputBuffer;
private Status lastStatus = Status.NEEDS_MORE_INPUT;
Wrapper(int inputBufferSize, ByteBuffer customDictionary) throws IOException {
if (customDictionary != null && !customDictionary.isDirect()) {
throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer");
}
Wrapper(int inputBufferSize) throws IOException {
this.context[1] = inputBufferSize;
this.inputBuffer = nativeCreate(this.context, customDictionary);
this.inputBuffer = nativeCreate(this.context);
if (this.context[0] == 0) {
throw new IOException("failed to initialize native brotli decoder");
}

View File

@@ -15,8 +15,6 @@ namespace {
typedef struct DecoderHandle {
BrotliDecoderState* state;
jobject custom_dictionary_ref;
uint8_t* input_start;
size_t input_offset;
size_t input_length;
@@ -44,7 +42,7 @@ extern "C" {
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) {
JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
bool ok = true;
DecoderHandle* handle = nullptr;
jlong context[2];
@@ -55,7 +53,6 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
ok = !!handle;
if (ok) {
handle->custom_dictionary_ref = nullptr;
handle->input_offset = 0;
handle->input_length = 0;
handle->input_start = nullptr;
@@ -73,36 +70,11 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
ok = !!handle->state;
}
if (ok && !!custom_dictionary) {
handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary);
if (!!handle->custom_dictionary_ref) {
uint8_t* custom_dictionary_address = static_cast<uint8_t*>(
env->GetDirectBufferAddress(handle->custom_dictionary_ref));
if (!!custom_dictionary_address) {
jlong capacity =
env->GetDirectBufferCapacity(handle->custom_dictionary_ref);
ok = (capacity > 0) && (capacity < (1 << 24));
if (ok) {
size_t custom_dictionary_size = static_cast<size_t>(capacity);
BrotliDecoderSetCustomDictionary(
handle->state, custom_dictionary_size, custom_dictionary_address);
}
} else {
ok = false;
}
} else {
ok = false;
}
}
if (ok) {
/* TODO: future versions (e.g. when 128-bit architecture comes)
might require thread-safe cookie<->handle mapping. */
context[0] = reinterpret_cast<jlong>(handle);
} else if (!!handle) {
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
if (!!handle->input_start) delete[] handle->input_start;
delete handle;
}
@@ -216,9 +188,6 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy(
env->GetLongArrayRegion(ctx, 0, 2, context);
DecoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
BrotliDecoderDestroyInstance(handle->state);
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
delete[] handle->input_start;
delete handle;
}

View File

@@ -77,22 +77,3 @@ java_test(
"@junit_junit//jar",
],
)
java_test(
name = "UseCustomDictionaryTest",
size = "large",
srcs = ["UseCustomDictionaryTest.java"],
data = [
":test_bundle",
"//:jni", # Bazel JNI workaround
],
jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"],
shard_count = 15,
deps = [
":enc",
"//java/org/brotli/integration:bundle_helper",
"//java/org/brotli/wrapper/common",
"//java/org/brotli/wrapper/dec",
"@junit_junit//jar",
],
)

View File

@@ -26,16 +26,10 @@ public class BrotliEncoderChannel extends Encoder implements WritableByteChannel
* @param destination underlying destination
* @param params encoding settings
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params,
int bufferSize, ByteBuffer customDictionary) throws IOException {
super(destination, params, bufferSize, customDictionary);
}
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params,
int bufferSize) throws IOException {
super(destination, params, bufferSize, null);
super(destination, params, bufferSize);
}
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params)

View File

@@ -8,7 +8,6 @@ package org.brotli.wrapper.enc;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
/**
@@ -26,17 +25,10 @@ public class BrotliOutputStream extends OutputStream {
* @param destination underlying destination
* @param params encoding settings
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize,
ByteBuffer customDictionary) throws IOException {
this.encoder = new Encoder(
Channels.newChannel(destination), params, bufferSize, customDictionary);
}
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize)
throws IOException {
this.encoder = new Encoder(Channels.newChannel(destination), params, bufferSize, null);
this.encoder = new Encoder(Channels.newChannel(destination), params, bufferSize);
}
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params)

View File

@@ -65,8 +65,8 @@ public class Encoder {
* @param params encoding parameters
* @param inputBufferSize read buffer size
*/
Encoder(WritableByteChannel destination, Parameters params, int inputBufferSize,
ByteBuffer customDictionary) throws IOException {
Encoder(WritableByteChannel destination, Parameters params, int inputBufferSize)
throws IOException {
if (inputBufferSize <= 0) {
throw new IllegalArgumentException("buffer size must be positive");
}
@@ -74,8 +74,7 @@ public class Encoder {
throw new NullPointerException("destination can not be null");
}
this.destination = destination;
this.encoder = new EncoderJNI.Wrapper(
inputBufferSize, params.quality, params.lgwin, customDictionary);
this.encoder = new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin);
this.inputBuffer = this.encoder.getInputBuffer();
}
@@ -157,8 +156,7 @@ public class Encoder {
* Encodes the given data buffer.
*/
public static byte[] compress(byte[] data, Parameters params) throws IOException {
EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(
data.length, params.quality, params.lgwin, null);
EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin);
ArrayList<byte[]> output = new ArrayList<byte[]>();
int totalOutputSize = 0;
try {

View File

@@ -13,7 +13,7 @@ import java.nio.ByteBuffer;
* JNI wrapper for brotli encoder.
*/
class EncoderJNI {
private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary);
private static native ByteBuffer nativeCreate(long[] context);
private static native void nativePush(long[] context, int length);
private static native ByteBuffer nativePull(long[] context);
private static native void nativeDestroy(long[] context);
@@ -28,15 +28,12 @@ class EncoderJNI {
protected final long[] context = new long[4];
private final ByteBuffer inputBuffer;
Wrapper(int inputBufferSize, int quality, int lgwin, ByteBuffer customDictionary)
Wrapper(int inputBufferSize, int quality, int lgwin)
throws IOException {
if (customDictionary != null && !customDictionary.isDirect()) {
throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer");
}
this.context[1] = inputBufferSize;
this.context[2] = quality;
this.context[3] = lgwin;
this.inputBuffer = nativeCreate(this.context, customDictionary);
this.inputBuffer = nativeCreate(this.context);
if (this.context[0] == 0) {
throw new IOException("failed to initialize native brotli encoder");
}

View File

@@ -1,104 +0,0 @@
package org.brotli.wrapper.enc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.brotli.integration.BundleHelper;
import org.brotli.wrapper.common.BrotliCommon;
import org.brotli.wrapper.dec.BrotliInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.junit.runner.RunWith;
import org.junit.runners.AllTests;
/** Tests for compression / decompression aided with LZ77 dictionary. */
@RunWith(AllTests.class)
public class UseCustomDictionaryTest {
// TODO: remove when Bazel get JNI support.
static {
System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")),
"liblibjni.so").getAbsolutePath());
}
static InputStream getBundle() throws IOException {
return new FileInputStream(System.getProperty("TEST_BUNDLE"));
}
/** Creates a test suite. */
public static TestSuite suite() throws IOException {
TestSuite suite = new TestSuite();
InputStream bundle = getBundle();
try {
List<String> entries = BundleHelper.listEntries(bundle);
for (String entry : entries) {
suite.addTest(new UseCustomDictionaryTestCase(entry));
}
} finally {
bundle.close();
}
return suite;
}
/** Test case with a unique name. */
static class UseCustomDictionaryTestCase extends TestCase {
final String entryName;
UseCustomDictionaryTestCase(String entryName) {
super("UseCustomDictionaryTest." + entryName);
this.entryName = entryName;
}
@Override
protected void runTest() throws Throwable {
UseCustomDictionaryTest.run(entryName);
}
}
private static void run(String entryName) throws Throwable {
InputStream bundle = getBundle();
byte[] original;
try {
original = BundleHelper.readEntry(bundle, entryName);
} finally {
bundle.close();
}
if (original == null) {
throw new RuntimeException("Can't read bundle entry: " + entryName);
}
ByteBuffer dictionary = BrotliCommon.makeNative(original);
ByteArrayOutputStream dst = new ByteArrayOutputStream();
OutputStream encoder = new BrotliOutputStream(dst,
new Encoder.Parameters().setQuality(11).setWindow(23), 1 << 23, dictionary);
try {
encoder.write(original);
} finally {
encoder.close();
}
byte[] compressed = dst.toByteArray();
// Just copy self from LZ77 dictionary -> ultimate compression ratio.
assertTrue(compressed.length < 80 + original.length / 65536);
InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed),
1 << 23, dictionary);
try {
long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original));
long crc = BundleHelper.fingerprintStream(decoder);
assertEquals(originalCrc, crc);
} finally {
decoder.close();
}
}
}

View File

@@ -15,8 +15,6 @@ namespace {
typedef struct EncoderHandle {
BrotliEncoderState* state;
jobject custom_dictionary_ref;
uint8_t* input_start;
size_t input_offset;
size_t input_last;
@@ -44,7 +42,7 @@ extern "C" {
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) {
JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
bool ok = true;
EncoderHandle* handle = nullptr;
jlong context[4];
@@ -55,7 +53,6 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
ok = !!handle;
if (ok) {
handle->custom_dictionary_ref = nullptr;
handle->input_offset = 0;
handle->input_last = 0;
handle->input_start = nullptr;
@@ -84,36 +81,11 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
}
}
if (ok && !!custom_dictionary) {
handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary);
if (!!handle->custom_dictionary_ref) {
uint8_t* custom_dictionary_address = static_cast<uint8_t*>(
env->GetDirectBufferAddress(handle->custom_dictionary_ref));
if (!!custom_dictionary_address) {
jlong capacity =
env->GetDirectBufferCapacity(handle->custom_dictionary_ref);
ok = (capacity > 0) && (capacity < (1 << 24));
if (ok) {
size_t custom_dictionary_size = static_cast<size_t>(capacity);
BrotliEncoderSetCustomDictionary(
handle->state, custom_dictionary_size, custom_dictionary_address);
}
} else {
ok = false;
}
} else {
ok = false;
}
}
if (ok) {
/* TODO: future versions (e.g. when 128-bit architecture comes)
might require thread-safe cookie<->handle mapping. */
context[0] = reinterpret_cast<jlong>(handle);
} else if (!!handle) {
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
if (!!handle->input_start) delete[] handle->input_start;
delete handle;
}
@@ -212,9 +184,6 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeDestroy(
env->GetLongArrayRegion(ctx, 0, 2, context);
EncoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
BrotliEncoderDestroyInstance(handle->state);
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
delete[] handle->input_start;
delete handle;
}