Update API, and more (#581)

Update API, and more:
 * remove "custom dictionary" support
 * c/encoder: fix #580: big-endian build
 * Java: reduce jar size
 * Java: speedup decoding
 * Java: add 32-bit CPU support
 * Java: make source code JS transpiler-ready

python/_brotli.cc

@@ -125,7 +125,7 @@ PyDoc_STRVAR(brotli_Compressor_doc,
 "An object to compress a byte string.\n"
 "\n"
 "Signature:\n"
-"  Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, dictionary='')\n"
+"  Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0)\n"
 "\n"
 "Args:\n"
 "  mode (int, optional): The compression mode can be MODE_GENERIC (default),\n"
@@ -138,8 +138,6 @@ PyDoc_STRVAR(brotli_Compressor_doc,
 "  lgblock (int, optional): Base 2 logarithm of the maximum input block size.\n"
 "    Range is 16 to 24. If set to 0, the value will be set based on the\n"
 "    quality. Defaults to 0.\n"
-"  dictionary (bytes, optional): Custom dictionary. Only last sliding window\n"
-"     size bytes will be used.\n"
 "\n"
 "Raises:\n"
 "  brotli.error: If arguments are invalid.\n");
@@ -174,20 +172,16 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj
   int quality = -1;
   int lgwin = -1;
   int lgblock = -1;
-  uint8_t* custom_dictionary = NULL;
-  size_t custom_dictionary_length = 0;
   int ok;
 
-  static const char *kwlist[] = {
-      "mode", "quality", "lgwin", "lgblock", "dictionary", NULL};
+  static const char *kwlist[] = {"mode", "quality", "lgwin", "lgblock", NULL};
 
-  ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&s#:Compressor",
+  ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&:Compressor",
                     const_cast<char **>(kwlist),
                     &mode_convertor, &mode,
                     &quality_convertor, &quality,
                     &lgwin_convertor, &lgwin,
-                    &lgblock_convertor, &lgblock,
-                    &custom_dictionary, &custom_dictionary_length);
+                    &lgblock_convertor, &lgblock);
   if (!ok)
     return -1;
   if (!self->enc)
@@ -202,15 +196,6 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj
   if (lgblock != -1)
     BrotliEncoderSetParameter(self->enc, BROTLI_PARAM_LGBLOCK, (uint32_t)lgblock);
 
-  if (custom_dictionary_length != 0) {
-    /* Unlike decoder, encoder processes dictionary immediately, that is why
-       it makes sense to release python GIL. */
-    Py_BEGIN_ALLOW_THREADS
-    BrotliEncoderSetCustomDictionary(self->enc, custom_dictionary_length,
-                                     custom_dictionary);
-    Py_END_ALLOW_THREADS
-  }
-
   return 0;
 }
 
@@ -432,11 +417,7 @@ PyDoc_STRVAR(brotli_Decompressor_doc,
 "An object to decompress a byte string.\n"
 "\n"
 "Signature:\n"
-"  Decompressor(dictionary='')\n"
-"\n"
-"Args:\n"
-"  dictionary (bytes, optional): Custom dictionary. Only last sliding window\n"
-"     size bytes will be used.\n"
+"  Decompressor()\n"
 "\n"
 "Raises:\n"
 "  brotli.error: If arguments are invalid.\n");
@@ -467,26 +448,17 @@ static PyObject* brotli_Decompressor_new(PyTypeObject *type, PyObject *args, PyO
 }
 
 static int brotli_Decompressor_init(brotli_Decompressor *self, PyObject *args, PyObject *keywds) {
-  uint8_t* custom_dictionary = NULL;
-  size_t custom_dictionary_length = 0;
   int ok;
 
-  static const char *kwlist[] = {
-      "dictionary", NULL};
+  static const char *kwlist[] = {NULL};
 
-  ok = PyArg_ParseTupleAndKeywords(args, keywds, "|s#:Decompressor",
-                    const_cast<char **>(kwlist),
-                    &custom_dictionary, &custom_dictionary_length);
+  ok = PyArg_ParseTupleAndKeywords(args, keywds, "|:Decompressor",
+                    const_cast<char **>(kwlist));
   if (!ok)
     return -1;
   if (!self->dec)
     return -1;
 
-  if (custom_dictionary_length != 0) {
-    BrotliDecoderSetCustomDictionary(self->dec, custom_dictionary_length,
-                                     custom_dictionary);
-  }
-
   return 0;
 }
 
@@ -644,8 +616,6 @@ PyDoc_STRVAR(brotli_decompress__doc__,
 "\n"
 "Args:\n"
 "  string (bytes): The compressed input data.\n"
-"  dictionary (bytes, optional): Custom dictionary. MUST be the same data\n"
-"     as passed to compress method.\n"
 "\n"
 "Returns:\n"
 "  The decompressed byte string.\n"
@@ -655,19 +625,15 @@ PyDoc_STRVAR(brotli_decompress__doc__,
 
 static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *keywds) {
   PyObject *ret = NULL;
-  const uint8_t *input, *custom_dictionary;
-  size_t length, custom_dictionary_length;
+  const uint8_t *input;
+  size_t length;
   int ok;
 
-  static const char *kwlist[] = {"string", "dictionary", NULL};
+  static const char *kwlist[] = {"string", NULL};
 
-  custom_dictionary = NULL;
-  custom_dictionary_length = 0;
-
-  ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|s#:decompress",
+  ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|:decompress",
                         const_cast<char **>(kwlist),
-                        &input, &length,
-                        &custom_dictionary, &custom_dictionary_length);
+                        &input, &length);
   if (!ok)
     return NULL;
 
@@ -677,9 +643,6 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key
   Py_BEGIN_ALLOW_THREADS
 
   BrotliDecoderState* state = BrotliDecoderCreateInstance(0, 0, 0);
-  if (custom_dictionary_length != 0) {
-    BrotliDecoderSetCustomDictionary(state, custom_dictionary_length, custom_dictionary);
-  }
 
   BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
   while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {

python/bro.py

@@ -114,13 +114,6 @@ def main(args=None):
         help='Base 2 logarithm of the maximum input block size. '
         'Range is 16 to 24. If set to 0, the value will be set based '
         'on the quality. Defaults to 0.')
-    params.add_argument(
-        '--custom-dictionary',
-        metavar='FILE',
-        type=str,
-        dest='dictfile',
-        help='Custom dictionary file.',
-        default=None)
     # set default values using global DEFAULT_PARAMS dictionary
     parser.set_defaults(**DEFAULT_PARAMS)
 
@@ -145,25 +138,16 @@ def main(args=None):
     else:
         outfile = get_binary_stdio('stdout')
 
-    if options.dictfile:
-        if not os.path.isfile(options.dictfile):
-            parser.error('file "%s" not found' % options.dictfile)
-        with open(options.dictfile, 'rb') as dictfile:
-            custom_dictionary = dictfile.read()
-    else:
-        custom_dictionary = ''
-
     try:
         if options.decompress:
-            data = brotli.decompress(data, dictionary=custom_dictionary)
+            data = brotli.decompress(data)
         else:
             data = brotli.compress(
                 data,
                 mode=options.mode,
                 quality=options.quality,
                 lgwin=options.lgwin,
-                lgblock=options.lgblock,
-                dictionary=custom_dictionary)
+                lgblock=options.lgblock)
     except brotli.error as e:
         parser.exit(1,
                     'bro: error: %s: %s' % (e, options.infile or 'sys.stdin'))

python/brotli.py

@@ -23,8 +23,7 @@ Compressor = _brotli.Compressor
 Decompressor = _brotli.Decompressor
 
 # Compress a byte string.
-def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
-             dictionary=''):
+def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0):
     """Compress a byte string.
 
     Args:
@@ -39,8 +38,6 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
       lgblock (int, optional): Base 2 logarithm of the maximum input block size.
         Range is 16 to 24. If set to 0, the value will be set based on the
         quality. Defaults to 0.
-      dictionary (bytes, optional): Custom dictionary. Only last sliding window
-        size bytes will be used.
 
     Returns:
       The compressed byte string.
@@ -49,7 +46,7 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
       brotli.error: If arguments are invalid, or compressor fails.
     """
     compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin,
-                            lgblock=lgblock, dictionary=dictionary)
+                            lgblock=lgblock)
     return compressor.process(string) + compressor.finish()
 
 # Decompress a compressed byte string.

python/tests/bro_test.py

@@ -62,8 +62,6 @@ class TestBroCompress(_test_utils.TestCase):
         temp_compressed = _test_utils.get_temp_compressed_name(test_data)
         original = test_data
         args = [PYTHON, BRO, '-f', '-d']
-        if 'dictionary' in kwargs:
-            args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
         args.extend(['-i', temp_compressed, '-o', temp_uncompressed])
         subprocess.check_call(args, env=TEST_ENV)
         self.assertFilesMatch(temp_uncompressed, original)
@@ -75,8 +73,6 @@ class TestBroCompress(_test_utils.TestCase):
             args.extend(['-q', str(kwargs['quality'])])
         if 'lgwin' in kwargs:
             args.extend(['--lgwin', str(kwargs['lgwin'])])
-        if 'dictionary' in kwargs:
-            args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
         args.extend(['-i', test_data, '-o', temp_compressed])
         subprocess.check_call(args, env=TEST_ENV)
 
@@ -87,8 +83,6 @@ class TestBroCompress(_test_utils.TestCase):
             args.extend(['-q', str(kwargs['quality'])])
         if 'lgwin' in kwargs:
             args.extend(['--lgwin', str(kwargs['lgwin'])])
-        if 'dictionary' in kwargs:
-            args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
         with open(temp_compressed, 'wb') as out_file:
             with open(test_data, 'rb') as in_file:
                 subprocess.check_call(
@@ -102,16 +96,6 @@ class TestBroCompress(_test_utils.TestCase):
         self._compress_pipe(test_data, **kwargs)
         self._check_decompression(test_data)
 
-    def _test_compress_file_custom_dictionary(self, test_data, **kwargs):
-        kwargs['dictionary'] = test_data
-        self._compress_file(test_data, **kwargs)
-        self._check_decompression(test_data, **kwargs)
-
-    def _test_compress_pipe_custom_dictionary(self, test_data, **kwargs):
-        kwargs['dictionary'] = test_data
-        self._compress_pipe(test_data, **kwargs)
-        self._check_decompression(test_data, **kwargs)
-
 
 _test_utils.generate_test_methods(
     TestBroCompress, variants=TestBroCompress.VARIANTS)

python/tests/compress_test.py

@@ -14,11 +14,7 @@ class TestCompress(_test_utils.TestCase):
     VARIANTS = {'quality': (1, 6, 9, 11), 'lgwin': (10, 15, 20, 24)}
 
     def _check_decompression(self, test_data, **kwargs):
-        # Only dictionary is supported as a kwarg to brotli.decompress.
-        if 'dictionary' in kwargs:
-            kwargs = {'dictionary': kwargs['dictionary']}
-        else:
-            kwargs = {}
+        kwargs = {}
         # Write decompression to temp file and verify it matches the original.
         temp_uncompressed = _test_utils.get_temp_uncompressed_name(test_data)
         temp_compressed = _test_utils.get_temp_compressed_name(test_data)
@@ -38,13 +34,6 @@ class TestCompress(_test_utils.TestCase):
         self._compress(test_data, **kwargs)
         self._check_decompression(test_data, **kwargs)
 
-    def _test_compress_custom_dictionary(self, test_data, **kwargs):
-        with open(test_data, 'rb') as in_file:
-            dictionary = in_file.read()
-        kwargs['dictionary'] = dictionary
-        self._compress(test_data, **kwargs)
-        self._check_decompression(test_data, **kwargs)
-
 
 _test_utils.generate_test_methods(TestCompress, variants=TestCompress.VARIANTS)