From 2a01fd8f31309262b4d1cd1901055883198c0fb2 Mon Sep 17 00:00:00 2001 From: gorloffslava Date: Mon, 29 Jul 2024 20:05:34 +0500 Subject: [PATCH] + Added ability to build Brotli Python bindings against system-provided brotli instead of vendored one --- pyproject.toml | 2 +- python/README.md | 10 +++ setup.py | 228 +++++++++++++++++++++++++++-------------------- 3 files changed, 142 insertions(+), 98 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fed528d..3725998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "pkgconfig"] build-backend = "setuptools.build_meta" diff --git a/python/README.md b/python/README.md index 4b6f63f..d67cd37 100644 --- a/python/README.md +++ b/python/README.md @@ -17,6 +17,16 @@ following command from this directory: $ make install +If you already have native Brotli installed on your system and want to use this one instead of the vendored sources, you +should set the `USE_SYSTEM_BROTLI=1` environment variable when building the wheel, like this: + + $ USE_SYSTEM_BROTLI=1 pip install brotli --no-binary brotli + +Brotli is found via the `pkg-config` utility. Moreover, you must build all 3 `brotlicommon`, `brotlienc`, and `brotlidec` +components. If you're installing brotli from the package manager, you need the development package, like this on Fedora: + + $ dnf install brotli brotli-devel + ### Development You may run the following commands from this directory: diff --git a/setup.py b/setup.py index 6cd325d..b3a434c 100644 --- a/setup.py +++ b/setup.py @@ -19,10 +19,23 @@ from distutils import errors from distutils import dep_util from distutils import log +from pkgconfig import configure_extension as pkgconfig_configure_extension + CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +def bool_from_environ(key: str): + value = os.environ.get(key) + if not value: + return False + if value == "1": + return True + if value == "0": + return False + raise ValueError(f"Environment variable {key} has invalid value {value}. Please set it to 1, 0 or an empty string") + + def read_define(path, macro): """ Return macro value from the given file. """ with open(path, 'r') as f: @@ -172,103 +185,124 @@ PACKAGE_DIR = {'': 'python'} PY_MODULES = ['brotli'] -EXT_MODULES = [ - Extension( - '_brotli', - sources=[ - 'python/_brotli.c', - 'c/common/constants.c', - 'c/common/context.c', - 'c/common/dictionary.c', - 'c/common/platform.c', - 'c/common/shared_dictionary.c', - 'c/common/transform.c', - 'c/dec/bit_reader.c', - 'c/dec/decode.c', - 'c/dec/huffman.c', - 'c/dec/state.c', - 'c/enc/backward_references.c', - 'c/enc/backward_references_hq.c', - 'c/enc/bit_cost.c', - 'c/enc/block_splitter.c', - 'c/enc/brotli_bit_stream.c', - 'c/enc/cluster.c', - 'c/enc/command.c', - 'c/enc/compound_dictionary.c', - 'c/enc/compress_fragment.c', - 'c/enc/compress_fragment_two_pass.c', - 'c/enc/dictionary_hash.c', - 'c/enc/encode.c', - 'c/enc/encoder_dict.c', - 'c/enc/entropy_encode.c', - 'c/enc/fast_log.c', - 'c/enc/histogram.c', - 'c/enc/literal_cost.c', - 'c/enc/memory.c', - 'c/enc/metablock.c', - 'c/enc/static_dict.c', - 'c/enc/utf8_util.c', - ], - depends=[ - 'c/common/constants.h', - 'c/common/context.h', - 'c/common/dictionary.h', - 'c/common/platform.h', - 'c/common/shared_dictionary_internal.h', - 'c/common/transform.h', - 'c/common/version.h', - 'c/dec/bit_reader.h', - 'c/dec/huffman.h', - 'c/dec/prefix.h', - 'c/dec/state.h', - 'c/enc/backward_references.h', - 'c/enc/backward_references_hq.h', - 'c/enc/backward_references_inc.h', - 'c/enc/bit_cost.h', - 'c/enc/bit_cost_inc.h', - 'c/enc/block_encoder_inc.h', - 'c/enc/block_splitter.h', - 'c/enc/block_splitter_inc.h', - 'c/enc/brotli_bit_stream.h', - 'c/enc/cluster.h', - 'c/enc/cluster_inc.h', - 'c/enc/command.h', - 'c/enc/compound_dictionary.h', - 'c/enc/compress_fragment.h', - 'c/enc/compress_fragment_two_pass.h', - 'c/enc/dictionary_hash.h', - 'c/enc/encoder_dict.h', - 'c/enc/entropy_encode.h', - 'c/enc/entropy_encode_static.h', - 'c/enc/fast_log.h', - 'c/enc/find_match_length.h', - 'c/enc/hash.h', - 'c/enc/hash_composite_inc.h', - 'c/enc/hash_forgetful_chain_inc.h', - 'c/enc/hash_longest_match64_inc.h', - 'c/enc/hash_longest_match_inc.h', - 'c/enc/hash_longest_match_quickly_inc.h', - 'c/enc/hash_rolling_inc.h', - 'c/enc/hash_to_binary_tree_inc.h', - 'c/enc/histogram.h', - 'c/enc/histogram_inc.h', - 'c/enc/literal_cost.h', - 'c/enc/memory.h', - 'c/enc/metablock.h', - 'c/enc/metablock_inc.h', - 'c/enc/params.h', - 'c/enc/prefix.h', - 'c/enc/quality.h', - 'c/enc/ringbuffer.h', - 'c/enc/static_dict.h', - 'c/enc/static_dict_lut.h', - 'c/enc/utf8_util.h', - 'c/enc/write_bits.h', - ], - include_dirs=[ - 'c/include', - ]), -] +USE_SYSTEM_BROTLI = bool_from_environ('USE_SYSTEM_BROTLI') +print(f"[EUGO] USE_SYSTEM_BROTLI={USE_SYSTEM_BROTLI}") +if USE_SYSTEM_BROTLI: + brotli_extension = Extension( + '_brotli', + sources=[ + 'python/_brotli.c' + ] + ) + + REQUIRED_BROTLI_SYSTEM_LIBRARIES = ["libbrotlicommon", "libbrotlienc", "libbrotlidec"] + pkgconfig_configure_extension(brotli_extension, " ".join(REQUIRED_BROTLI_SYSTEM_LIBRARIES)) + + print(f"[EUGO] {brotli_extension.include_dirs}") + print(f"[EUGO] {brotli_extension.library_dirs}") + print(f"[EUGO] {brotli_extension.libraries}") + print(f"[EUGO] {brotli_extension.extra_compile_args}") + print(f"[EUGO] {brotli_extension.extra_link_args}") + + EXT_MODULES = [brotli_extension] +else: + EXT_MODULES = [ + Extension( + '_brotli', + sources=[ + 'python/_brotli.c', + 'c/common/constants.c', + 'c/common/context.c', + 'c/common/dictionary.c', + 'c/common/platform.c', + 'c/common/shared_dictionary.c', + 'c/common/transform.c', + 'c/dec/bit_reader.c', + 'c/dec/decode.c', + 'c/dec/huffman.c', + 'c/dec/state.c', + 'c/enc/backward_references.c', + 'c/enc/backward_references_hq.c', + 'c/enc/bit_cost.c', + 'c/enc/block_splitter.c', + 'c/enc/brotli_bit_stream.c', + 'c/enc/cluster.c', + 'c/enc/command.c', + 'c/enc/compound_dictionary.c', + 'c/enc/compress_fragment.c', + 'c/enc/compress_fragment_two_pass.c', + 'c/enc/dictionary_hash.c', + 'c/enc/encode.c', + 'c/enc/encoder_dict.c', + 'c/enc/entropy_encode.c', + 'c/enc/fast_log.c', + 'c/enc/histogram.c', + 'c/enc/literal_cost.c', + 'c/enc/memory.c', + 'c/enc/metablock.c', + 'c/enc/static_dict.c', + 'c/enc/utf8_util.c', + ], + depends=[ + 'c/common/constants.h', + 'c/common/context.h', + 'c/common/dictionary.h', + 'c/common/platform.h', + 'c/common/shared_dictionary_internal.h', + 'c/common/transform.h', + 'c/common/version.h', + 'c/dec/bit_reader.h', + 'c/dec/huffman.h', + 'c/dec/prefix.h', + 'c/dec/state.h', + 'c/enc/backward_references.h', + 'c/enc/backward_references_hq.h', + 'c/enc/backward_references_inc.h', + 'c/enc/bit_cost.h', + 'c/enc/bit_cost_inc.h', + 'c/enc/block_encoder_inc.h', + 'c/enc/block_splitter.h', + 'c/enc/block_splitter_inc.h', + 'c/enc/brotli_bit_stream.h', + 'c/enc/cluster.h', + 'c/enc/cluster_inc.h', + 'c/enc/command.h', + 'c/enc/compound_dictionary.h', + 'c/enc/compress_fragment.h', + 'c/enc/compress_fragment_two_pass.h', + 'c/enc/dictionary_hash.h', + 'c/enc/encoder_dict.h', + 'c/enc/entropy_encode.h', + 'c/enc/entropy_encode_static.h', + 'c/enc/fast_log.h', + 'c/enc/find_match_length.h', + 'c/enc/hash.h', + 'c/enc/hash_composite_inc.h', + 'c/enc/hash_forgetful_chain_inc.h', + 'c/enc/hash_longest_match64_inc.h', + 'c/enc/hash_longest_match_inc.h', + 'c/enc/hash_longest_match_quickly_inc.h', + 'c/enc/hash_rolling_inc.h', + 'c/enc/hash_to_binary_tree_inc.h', + 'c/enc/histogram.h', + 'c/enc/histogram_inc.h', + 'c/enc/literal_cost.h', + 'c/enc/memory.h', + 'c/enc/metablock.h', + 'c/enc/metablock_inc.h', + 'c/enc/params.h', + 'c/enc/prefix.h', + 'c/enc/quality.h', + 'c/enc/ringbuffer.h', + 'c/enc/static_dict.h', + 'c/enc/static_dict_lut.h', + 'c/enc/utf8_util.h', + 'c/enc/write_bits.h', + ], + include_dirs=[ + 'c/include', + ]), + ] TEST_SUITE = 'setup.get_test_suite'