Compare commits

..

1 Commits

Author SHA1 Message Date
Eugene Kliuchnikov
66c14517cf Update version to 0.5.2 2016-08-11 17:47:18 +02:00
395 changed files with 38588 additions and 82655 deletions

View File

@@ -1,6 +0,0 @@
# Exclude Bazel roots (workspaces)
c/fuzz
go
java
js
research

View File

@@ -1,40 +0,0 @@
# http://editorconfig.org
# Consistent coding style across different editors.
# Top-most file
root = true
# Global styles:
# - indent 2 spaces
# - add final new line
# - trim trailing whitespace
[*]
charset = utf-8
end_of_line = lf
indent_size = 2
indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true
# BUILD:
# - indent 4 spaces
[BUILD]
indent_size = 4
# Makefile:
# - indent 1 tab
[Makefile]
indent_size = tab
indent_style = tab
# Markdown:
# - indent 4 spaces
# - trailing whitespace is significant
[*.md]
indent_size = 4
trim_trailing_whitespace = false
# Python
# - indent 4 spaces
[*.py]
indent_size = 4

54
.gitattributes vendored
View File

@@ -1,54 +0,0 @@
tests/testdata/* binary
# Exclude everything
**/** export-ignore
# Add top-level files
.bazelignore !export-ignore
BUILD.bazel !export-ignore
CHANGELOG.md !export-ignore
CMakeLists.txt !export-ignore
CONTRIBUTING.md !export-ignore
LICENSE !export-ignore
MANIFEST.in !export-ignore
README !export-ignore
README.md !export-ignore
SECURITY.md !export-ignore
setup.cfg !export-ignore
setup.py !export-ignore
WORKSPACE.bazel !export-ignore
# Add sources
c !export-ignore
c/** !export-ignore
c/common/dictionary.bin* export-ignore
c/fuzz export-ignore
# Add man pages
docs !export-ignore
docs/** !export-ignore
docs/brotli-comparison-study-2015-09-22.pdf export-ignore
# Add python bindings + tests
python !export-ignore
python/** !export-ignore
# Add go bindings + tests
go !export-ignore
go/** !export-ignore
# Add more build files.
scripts !export-ignore
scripts/download_testdata.sh !export-ignore
scripts/libbrotli*.pc.in !export-ignore
# Add testdata
tests !export-ignore
tests/*.sh !export-ignore
tests/*.cmake !export-ignore
tests/testdata !export-ignore
tests/testdata/empty !export-ignore
tests/testdata/empty.compressed !export-ignore
tests/testdata/ukkonooa !export-ignore
tests/testdata/ukkonooa.compressed !export-ignore
tests/testdata/zerosukkanooa.compressed !export-ignore

View File

@@ -1,11 +0,0 @@
# Copyright 2023 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"

View File

@@ -1,355 +0,0 @@
# Copyright 2021 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for building and running tests under Ubuntu
name: Build/Test
on:
push:
branches:
- master
pull_request:
types: [opened, reopened, labeled, unlabeled, synchronize]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build_test:
name: Build and test ${{ matrix.name }}
runs-on: ${{ matrix.os || 'ubuntu-latest' }}
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix:
include:
- name: cmake:gcc
build_system: cmake
c_compiler: gcc
cxx_compiler: g++
- name: cmake:gcc-old
build_system: cmake
c_compiler: gcc
cxx_compiler: g++
os: ubuntu-22.04
- name: cmake:clang
build_system: cmake
c_compiler: clang
cxx_compiler: clang
- name: cmake:clang-old
build_system: cmake
c_compiler: clang
cxx_compiler: clang
os: ubuntu-22.04
- name: cmake:package
build_system: cmake
cmake_args: -DBROTLI_BUILD_FOR_PACKAGE=ON
- name: cmake:static
build_system: cmake
cmake_args: -DBUILD_SHARED_LIBS=OFF
- name: cmake:clang:asan
build_system: cmake
sanitizer: address
c_compiler: clang
cxx_compiler: clang++
- name: cmake:clang:tsan
build_system: cmake
sanitizer: thread
c_compiler: clang
cxx_compiler: clang++
- name: cmake:clang:ubsan
build_system: cmake
sanitizer: undefined
c_compiler: clang
cxx_compiler: clang++
c_flags: -fno-sanitize-recover=undefined,integer
- name: cmake:qemu-arm-neon-gcc
build_system: cmake
c_compiler: arm-linux-gnueabihf-gcc
cxx_compiler: arm-linux-gnueabihf-g++
c_flags: -march=armv7-a -mfloat-abi=hard -mfpu=neon
extra_apt_pkgs: gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
- name: cmake-osx:clang
build_system: cmake
c_compiler: clang
cxx_compiler: clang++
os: macos-latest
- name: cmake-osx:gcc
build_system: cmake
c_compiler: gcc
cxx_compiler: g++
os: macos-latest
- name: cmake-ios:clang
build_system: cmake
c_compiler: clang
cxx_compiler: clang++
os: macos-latest
skip_tests: true # TODO(eustas): run tests in a simulator
cmake_args: >-
-DCMAKE_SYSTEM_NAME=iOS
-DCMAKE_OSX_ARCHITECTURES=arm64
- name: cmake-win64:msvc-rel
build_system: cmake
cmake_generator: Visual Studio 17 2022
cmake_config: Release
os: windows-latest
- name: cmake-win64:msvc-dbg
build_system: cmake
cmake_generator: Visual Studio 17 2022
cmake_config: Debug
os: windows-latest
- name: fuzz:clang
build_system: fuzz
c_compiler: clang
cxx_compiler: clang++
- name: python3.10:clang
build_system: python
python_version: "3.10"
c_compiler: clang
cxx_compiler: clang++
- name: python3.10-win
build_system: python
python_version: "3.10"
# TODO: investigate why win-builds can't run tests
os: windows-2022
- name: maven
build_system: maven
- name: bazel:root
build_system: bazel
bazel_project: .
- name: bazel:go
build_system: bazel
bazel_project: go
- name: bazel:java
build_system: bazel
bazel_project: java
- name: bazel:research
build_system: bazel
bazel_project: research
- name: bazel-osx:root
build_system: bazel
bazel_project: .
os: macos-latest
- name: bazel-osx:go
build_system: bazel
bazel_project: go
os: macos-latest
- name: bazel-osx:java
build_system: bazel
bazel_project: java
os: macos-latest
- name: bazel-osx:research
build_system: bazel
bazel_project: research
os: macos-latest
- name: bazel-win:root
build_system: bazel
bazel_project: .
os: windows-latest
# TODO(eustas): restore when go is fixed
#- name: bazel-win:go
# build_system: bazel
# bazel_project: go
# os: windows-latest
# TODO(eustas): restore when kotlin is fixed
#- name: bazel-win:java
# build_system: bazel
# bazel_project: java
# os: windows-latest
- name: bazel-win:research
build_system: bazel
bazel_project: research
os: windows-latest
env:
CC: ${{ matrix.c_compiler || 'gcc' }}
CXX: ${{ matrix.cxx_compiler || 'gcc' }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Install extra deps @ Ubuntu
if: ${{ runner.os == 'Linux' }}
# Already installed: bazel, clang{13-15}, cmake, gcc{9.5-13.1}, java{8,11,17,21}, maven, python{3.10}
run: |
EXTRA_PACKAGES="${{ matrix.extra_apt_pkgs || '' }}"
sudo apt update
sudo apt install -y ${EXTRA_PACKAGES}
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- name: Configure / Build / Test with CMake
if: ${{ matrix.build_system == 'cmake' }}
run: |
export ASAN_OPTIONS=detect_leaks=0
declare -a CMAKE_OPTIONS=(${{ matrix.cmake_args || '' }})
CMAKE_OPTIONS+=("-DCMAKE_VERBOSE_MAKEFILE=ON")
[ ! -z '${{ matrix.c_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_C_COMPILER='${{ matrix.c_compiler }}')
[ ! -z '${{ matrix.cxx_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_CXX_COMPILER='${{ matrix.cxx_compiler }}')
[ ! -z '${{ matrix.sanitizer || '' }}' ] && CMAKE_OPTIONS+=(-DENABLE_SANITIZER='${{ matrix.sanitizer }}')
[ ! -z '${{ matrix.cmake_generator || '' }}' ] && export CMAKE_GENERATOR='${{ matrix.cmake_generator }}'
declare -a CMAKE_BUILD_OPTIONS=()
[ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_BUILD_OPTIONS+=(--config '${{ matrix.cmake_config }}')
declare -a CMAKE_TEST_OPTIONS=()
[ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_TEST_OPTIONS+=(-C '${{ matrix.cmake_config }}')
cmake -B out . ${CMAKE_OPTIONS[*]} -DCMAKE_C_FLAGS='${{ matrix.c_flags || '' }}'
cmake --build out ${CMAKE_BUILD_OPTIONS[*]}
cd out
[ ! -z '${{ matrix.skip_tests || '' }}' ] || ctest ${CMAKE_TEST_OPTIONS[*]}
cd ..
- name: Quick Fuzz
if: ${{ matrix.build_system == 'fuzz' }}
run: |
mkdir ${RUNNER_TEMP}/decode_corpora
unzip java/org/brotli/integration/fuzz_data.zip -d ${RUNNER_TEMP}/decode_corpora
cd ${GITHUB_WORKSPACE}/c/fuzz
bazelisk build --config=asan-libfuzzer :decode_fuzzer
for f in `ls ${RUNNER_TEMP}/decode_corpora`
do
echo "Testing $f"
./bazel-bin/decode_fuzzer_bin ${RUNNER_TEMP}/decode_corpora/$f
done
- name: Build with Bazel
if: ${{ matrix.build_system == 'bazel' }}
run: |
cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
bazelisk build -c opt ...:all --java_runtime_version=remotejdk_21
- name: Fix symlinks for Bazel (Windows)
if: ${{ matrix.build_system == 'bazel' && runner.os == 'Windows' && matrix.bazel_project == 'java' }}
shell: python
run: |
import fnmatch
import os
import os.path
from shutil import copyfile
os.chdir('${{ matrix.bazel_project }}')
print('Searching for manifests in ' + os.getcwd())
matches = []
for root, dirnames, filenames in os.walk('bazel-bin\\org\\brotli'):
for filename in fnmatch.filter(filenames, '*.runfiles_manifest'):
matches.append(os.path.join(root, filename))
for match in matches:
print('Scanning manifest ' + match)
runfiles = match[:-len('_manifest')]
with open(match) as manifest:
for entry in manifest:
entry = entry.strip()
if not entry.startswith("_main"):
continue
if entry.startswith("_main/external"):
continue
(alias, space, link) = entry.partition(' ')
if alias.endswith('.jar') or alias.endswith('.exe'):
continue
link = link.replace('/', '\\')
alias = alias.replace('/', '\\')
dst = os.path.join(runfiles, alias)
if not os.path.exists(dst):
print(link + ' -> ' + dst)
parent = os.path.dirname(dst)
if not os.path.exists(parent):
os.makedirs(parent)
copyfile(link, dst)
print('Finished resolving symlinks')
- name: Test with Bazel
if: ${{ matrix.build_system == 'bazel' }}
run: |
cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
bazelisk query "tests(...)" --output=label > ${RUNNER_TEMP}/tests.lst
[ -s ${RUNNER_TEMP}/tests.lst ] && bazelisk test -c opt ...:all --java_runtime_version=remotejdk_21
bazelisk clean
- name: Build / Test with Maven
if: ${{ matrix.build_system == 'maven' }}
run: |
export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
cd java/org/brotli
mvn -B install
# TODO(eustas): nuke maven build?
# cd integration
# mvn -B verify
- uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
if: ${{ matrix.build_system == 'python' }}
with:
python-version: ${{ matrix.python_version }}
# TODO(eustas): use modern setuptools (split out testing)
- name: Build / Test with Python
if: ${{ matrix.build_system == 'python' }}
run: |
python -VV
python -c "import sys; sys.exit('Invalid python version') if '.'.join(map(str,sys.version_info[0:2])) != '${{ matrix.python_version }}' else True"
pip install setuptools==51.3.3 pytest
python setup.py build_ext --inplace
pytest ./python/tests
build_test_dotnet:
name: Build and test with .NET
runs-on: ubuntu-latest
steps:
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- name: Build / Test
run: |
cd csharp
dotnet build brotlidec.csproj --configuration Release
dotnet test brotlidec.Tests.csproj

View File

@@ -1,70 +0,0 @@
# Copyright 2025 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for building and running tests with WASM
name: Build/Test WASM
on:
push:
branches:
- master
pull_request:
types: [opened, reopened, labeled, unlabeled, synchronize]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build_test_wasm:
name: Build and test with WASM
runs-on: ubuntu-latest
env:
CCACHE_DIR: ${{ github.workspace }}/.ccache
BUILD_TARGET: wasm32
EM_VERSION: 3.1.51
# As of 28.08.2025 ubuntu-latest is 24.04; it is shipped with node 22.18
NODE_VERSION: 22
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: true
fetch-depth: 1
- name: Install node
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
with:
node-version: ${{env.NODE_VERSION}}
- name: Get non-EMSDK node path
run: which node >> $HOME/.base_node_path
- name: Install emsdk
uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
with:
version: ${{env.EM_VERSION}}
no-cache: true
- name: Set EMSDK node version
run: |
echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EMSDK/.emscripten
emsdk construct_env
- name: Build
run: |
LDFLAGS=" -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 " emcmake cmake -B out .
cmake --build out
cd out; ctest --output-on-failure; cd ..

View File

@@ -1,81 +0,0 @@
name: "CodeQL"
on:
push:
branches: [ "master" ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ "master" ]
schedule:
- cron: '18 15 * * 0'
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
name: Analyze
runs-on: 'ubuntu-latest'
timeout-minutes: 360
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'cpp', 'java', 'javascript', 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v3.29.5
with:
languages: ${{ matrix.language }}
# CodeQL is currently crashing on files with large lists:
# https://github.com/github/codeql/issues/13656
config: |
paths-ignore:
- research
- js/test_data.*
- if: matrix.language == 'cpp'
name: Build CPP
uses: github/codeql-action/autobuild@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v3.29.5
- if: matrix.language == 'cpp' || matrix.language == 'java'
name: Build Java
run: |
cd ${GITHUB_WORKSPACE}/java
bazelisk build --spawn_strategy=local --nouse_action_cache -c opt ...:all
- if: matrix.language == 'javascript'
name: Build JS
uses: github/codeql-action/autobuild@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v3.29.5
- if: matrix.language == 'cpp' || matrix.language == 'python'
name: Build Python
run: |
python setup.py build_ext
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v3.29.5
with:
category: "/language:${{matrix.language}}"
ref: "${{ github.ref != 'master' && github.ref || '/refs/heads/master' }}"
sha: "${{ github.sha }}"

View File

@@ -1,47 +0,0 @@
# Copyright 2020 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for building / running oss-fuzz.
name: CIFuzz
on: [pull_request]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
Fuzzing:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Build Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@3e6a7fd7bcd631647ab9beed1fe0897498e6af39 # 22.09.2025
with:
oss-fuzz-project-name: 'brotli'
dry-run: false
- name: Run Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@3e6a7fd7bcd631647ab9beed1fe0897498e6af39 # 22.09.2025
with:
oss-fuzz-project-name: 'brotli'
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: failure()
with:
name: artifacts
path: ./out/artifacts

View File

@@ -1,55 +0,0 @@
# Copyright 2025 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for checking typos and buildifier, formatting, etc.
name: "Lint"
on:
push:
branches: [ "master" ]
pull_request:
branches: [ "master" ]
schedule:
- cron: '18 15 * * 0'
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
check:
name: Lint
runs-on: 'ubuntu-latest'
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
- name: Install tools
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
brew install buildifier ruff typos-cli
- name: Check typos
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
./scripts/check_typos.sh
- name: Lint Python code
run: |
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
ruff check --extend-select=C4,C90,PERF,RET,SIM,W
# TODO(eustas): run buildifier

View File

@@ -1,245 +0,0 @@
# Copyright 2023 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for building the release binaries.
name: Release build / deploy
on:
push:
branches:
- master
- v*.*.*
release:
types: [ published ]
pull_request:
types: [opened, reopened, labeled, unlabeled, synchronize]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
windows_build:
name: Windows Build (vcpkg / ${{ matrix.triplet }})
runs-on: ${{ matrix.runs_on }}
strategy:
fail-fast: false
matrix:
include:
- triplet: x86-windows-dynamic
arch: '-A Win32'
build_shared_libs: 'ON'
runs_on: windows-latest
- triplet: x64-windows-dynamic
arch: '-A x64'
build_shared_libs: 'ON'
runs_on: windows-latest
- triplet: arm64-windows-dynamic
arch: '-A ARM64'
build_shared_libs: 'ON'
runs_on: windows-11-arm
- triplet: x86-windows-static
arch: '-A Win32'
build_shared_libs: 'OFF'
runs_on: windows-latest
- triplet: x64-windows-static
arch: '-A x64'
build_shared_libs: 'OFF'
runs_on: windows-latest
- triplet: arm64-windows-static
arch: '-A ARM64'
build_shared_libs: 'OFF'
runs_on: windows-11-arm
env:
VCPKG_VERSION: '2022.11.14'
VCPKG_ROOT: vcpkg
VCPKG_DISABLE_METRICS: 1
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
id: cache-vcpkg
with:
path: vcpkg
key: release-${{ runner.os }}-vcpkg-${{ env.VCPKG_VERSION }}-${{ matrix.triplet }}
- name: Download vcpkg
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
shell: 'powershell'
run: |
Invoke-WebRequest -Uri "https://github.com/microsoft/vcpkg/archive/refs/tags/${{ env.VCPKG_VERSION }}.zip" -OutFile "vcpkg.zip"
- name: Bootstrap vcpkg
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
shell: 'bash'
run: |
set -x
unzip -q vcpkg.zip
rm -rf ${VCPKG_ROOT}
mv vcpkg-${VCPKG_VERSION} ${VCPKG_ROOT}
${VCPKG_ROOT}/bootstrap-vcpkg.sh
- name: Configure
shell: 'bash'
run: |
set -x
mkdir out
cmake -Bout -H. ${{ matrix.arch }} \
-DBUILD_TESTING=OFF \
-DBUILD_SHARED_LIBS=${{ matrix.build_shared_libs }} \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=`pwd`/prefix \
-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
-DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \
#
- name: Build
shell: 'bash'
run: |
set -x
cmake --build out --config Release
- name: Install
shell: 'bash'
run: |
set -x
cmake --build out --config Release --target install
cp LICENSE prefix/bin/LICENSE.brotli
- name: Package release zip
shell: 'powershell'
run: |
Compress-Archive -Path prefix\bin\* `
-DestinationPath brotli-${{matrix.triplet}}.zip
- name: Upload package
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: brotli-${{matrix.triplet}}
path: brotli-${{matrix.triplet}}.zip
compression-level: 0
testdata_upload:
name: Upload testdata
runs-on: 'ubuntu-latest'
defaults:
run:
shell: bash
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- name: Compress testdata
run: |
tar cvfJ testdata.txz tests/testdata
- name: Upload archive
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: testdata
path: testdata.txz
compression-level: 0
publish_release_assets:
name: Publish release assets
needs: [windows_build, testdata_upload]
if: github.event_name == 'release'
runs-on: [ubuntu-latest]
permissions:
contents: write
steps:
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- name: Download all artifacts
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
path: release_assets
merge-multiple: true
- name: Publish assets
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload ${{ github.event.release.tag_name }} ./release_assets/*
archive_build:
needs: publish_release_assets
name: Build and test from archive
runs-on: 'ubuntu-latest'
defaults:
run:
shell: bash
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: Checkout the source
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
submodules: false
fetch-depth: 1
- name: Archive
run: |
git archive HEAD -o archive.tgz
- name: Pick tag
run: |
echo "BROTLI_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
- name: Extract
run: |
mkdir archive
cd archive
tar xvzf ../archive.tgz
- name: Download testdata
run: |
cd archive
scripts/download_testdata.sh
- name: Configure and Build
run: |
cd archive
cmake -B out .
cmake --build out
- name: Test
run: |
cd archive
cd out
ctest

View File

@@ -1,82 +0,0 @@
# This workflow uses actions that are not certified by GitHub. They are provided
# by a third-party and are governed by separate terms of service, privacy
# policy, and support documentation.
name: Scorecard supply-chain security
on:
# For Branch-Protection check. Only the default branch is supported. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
branch_protection_rule:
# To guarantee Maintained check is occasionally updated. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
schedule:
- cron: '23 21 * * 1'
push:
branches: [ "master" ]
# Declare default permissions as read only.
permissions: read-all
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analysis:
name: Scorecard analysis
runs-on: ubuntu-latest
permissions:
# Needed to upload the results to code-scanning dashboard.
security-events: write
# Needed to publish results and get a badge (see publish_results below).
id-token: write
# Uncomment the permissions below if installing in a private repository.
# contents: read
# actions: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
with:
egress-policy: audit
- name: "Checkout code"
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
with:
results_file: results.sarif
results_format: sarif
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
# - you want to enable the Branch-Protection check on a *public* repository, or
# - you are installing Scorecard on a *private* repository
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
# Public repositories:
# - Publish results to OpenSSF REST API for easy access by consumers
# - Allows the repository to include the Scorecard badge.
# - See https://github.com/ossf/scorecard-action#publishing-results.
# For private repositories:
# - `publish_results` will always be set to `false`, regardless
# of the value entered here.
publish_results: true
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: SARIF file
path: results.sarif
retention-days: 5
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v2.23.3
with:
sarif_file: results.sarif

19
.gitignore vendored
View File

@@ -1,20 +1,9 @@
# C
*.d
*.o
*.obj
*.pyc
*.txt.uncompressed
*.bro
*.unbro
bin/
buildfiles/
**/obj/
dist/
**/bazel-*
# Python
__pycache__/
*.py[cod]
*.so
*.egg-info/
# Tests
*.txt.uncompressed
*.br
*.unbr

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "terryfy"]
path = terryfy
url = https://github.com/MacPython/terryfy.git

58
.travis.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/bin/bash
case "$1" in
"install")
case "${TRAVIS_OS_NAME}" in
"osx")
brew update
brew install binutils
case "${CC}" in
"gcc-"*)
which ${CC} || brew install homebrew/versions/gcc$(echo "${CC#*-}" | sed 's/\.//')
;;
esac
case "${BUILD_SYSTEM}" in
"python")
source terryfy/travis_tools.sh
get_python_environment $INSTALL_TYPE $PYTHON_VERSION venv
pip install --upgrade wheel
;;
esac
;;
esac
;;
"script")
case "${BUILD_SYSTEM}" in
"cmake")
mkdir builddir && cd builddir
CMAKE_FLAGS=
if [ "${CROSS_COMPILE}" = "yes" ]; then
CMAKE_FLAGS="-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_RC_COMPILER=${RC_COMPILER}"
fi
cmake ${CMAKE_FLAGS} -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_SANITIZER="${SANITIZER}" -DCMAKE_C_FLAGS="${CFLAGS}" ..
make VERBOSE=1
ctest -V
;;
"python")
if [ "${TRAVIS_OS_NAME}" = "osx" ]; then
source venv/bin/activate
fi
python setup.py build_ext test
;;
esac
;;
"after_success")
case "${BUILD_SYSTEM}" in
"python")
case "${TRAVIS_OS_NAME}" in
"osx")
source venv/bin/activate
pip wheel -w dist .
;;
esac
;;
esac
;;
esac

246
.travis.yml Normal file
View File

@@ -0,0 +1,246 @@
language: c
sudo: false
matrix:
include:
###
## Linux builds using various versions of GCC.
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-6 CXX_COMPILER=g++-6
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- g++-6
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.9
- g++-4.9
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.8 CXX_COMPILER=g++-4.8
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- g++-4.8
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.7 CXX_COMPILER=g++-4.7
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.7
- g++-4.7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.6 CXX_COMPILER=g++-4.6
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.6
- g++-4.6
# - os: linux
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.5 CXX_COMPILER=g++-4.5
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# packages:
# - gcc-4.5
# - g++-4.5
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.4 CXX_COMPILER=g++-4.4
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.4
- g++-4.4
###
## clang on Linux
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.8 CXX_COMPILER=clang++-3.8
addons:
apt:
sources:
- llvm-toolchain-precise-3.8
- ubuntu-toolchain-r-test
packages:
- clang-3.8
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.7 CXX_COMPILER=clang++-3.7
addons:
apt:
sources:
- llvm-toolchain-precise-3.7
- ubuntu-toolchain-r-test
packages:
- clang-3.7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.6 CXX_COMPILER=clang++-3.6
addons:
apt:
sources:
- llvm-toolchain-precise-3.6
- ubuntu-toolchain-r-test
packages:
- clang-3.6
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.5 CXX_COMPILER=clang++-3.5
addons:
apt:
sources:
- llvm-toolchain-precise-3.5
- ubuntu-toolchain-r-test
packages:
- clang-3.5
###
## Python build on Linux
###
- os: linux
env: BUILD_SYSTEM=python C_COMPILER=gcc-6 CXX_COMPILER=g++-6
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- g++-6
###
## CMake on OS X
##
## These all work, but it seems unnecessary to actually build them
## all since we already test all these versions of GCC on Linux.
## We'll just test 4.4 and the most recent version.
###
- os: osx
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-6 CXX_COMPILER=g++-6
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-5 CXX_COMPILER=g++-5
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.8 CXX_COMPILER=g++-4.8
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.7 CXX_COMPILER=g++-4.7
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.6 CXX_COMPILER=g++-4.6
# - os: osx
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.5 CXX_COMPILER=g++-4.5
- os: osx
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.4 CXX_COMPILER=g++-4.4
###
## Python OS X builds
###
- os: osx
env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=2.7.12
- os: osx
env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=3.4.4
- os: osx
env: BUILD_SYSTEM=python INSTALL_TYPE=macpython PYTHON_VERSION=3.5.2
###
## Sanitizers
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.8 CXX_COMPILER=clang++-3.8 SANITIZER=address
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.8 CXX_COMPILER=clang++-3.8 SANITIZER=thread
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.8 CXX_COMPILER=clang++-3.8 SANITIZER=undefined CFLAGS="-fno-sanitize-recover=undefined,integer"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
###
## mingw
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=x86_64-w64-mingw32-gcc CXX_COMPILER=x86_64-w64-mingw32-g++ RC_COMPILER=x86_64-w64-mingw32-windres CROSS_COMPILE=yes
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- george-edison55-precise-backports
packages:
- binutils-mingw-w64-x86-64
- g++-mingw-w64-x86-64
- gcc-mingw-w64-x86-64
- binutils-mingw-w64-x86-64
- wine
# Because 2.8.6 passes -rdynamic to the linker, which breaks the build.
- cmake
- cmake-data
before_install:
###
## If we use the matrix to set CC/CXX Travis, overwrites the values,
## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX
## here (after Travis has set CC/CXX).
###
- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
- if [ -n "${CXX_COMPILER}" ]; then export CXX="${CXX_COMPILER}"; fi
install:
- ./.travis.sh install
script:
- ./.travis.sh script
after_success:
- ./.travis.sh after_success
before_deploy:
- if [ "${BUILD_SYSTEM}" = "python" ] && [ "${TRAVIS_OS_NAME}" = "osx" ]; then export WHEELS=$(ls ./dist/*.whl); fi
deploy:
provider: releases
api_key:
secure: YcCBi6W/w4dtKCa59Wfm8L5lGWvK7KxaFNDr3yh1Hz5aStXXf758pEMHGewnlbfbwuj5a3SjBb1nLp1M69OQJfxm442uXBaBKo52PM9PPbD7NjvbNIso73pqcSODXQXKuZxDFpEhfuDTVq3hUkUqiwhChWhrFucJsSL51i7qSss=
file: "${WHEELS}"
skip_cleanup: true
on:
repo: "google/brotli"
tags: true
condition: "${BUILD_SYSTEM} = python && ${TRAVIS_OS_NAME} = osx"

90
BUILD Normal file
View File

@@ -0,0 +1,90 @@
# Description:
# Brotli is a generic-purpose lossless compression algorithm.
package(
default_visibility = ["//visibility:public"],
)
licenses(["notice"]) # MIT
STRICT_C_OPTIONS = [
"--pedantic-errors",
"-Wall",
"-Wconversion",
"-Werror",
"-Wextra",
"-Wlong-long",
"-Wmissing-declarations",
"-Wmissing-prototypes",
"-Wno-strict-aliasing",
"-Wshadow",
"-Wsign-compare",
]
filegroup(
name = "common_headers",
srcs = glob(["common/*.h"]),
)
filegroup(
name = "common_sources",
srcs = glob(["common/*.c"]),
)
filegroup(
name = "dec_headers",
srcs = glob(["dec/*.h"]),
)
filegroup(
name = "dec_sources",
srcs = glob(["dec/*.c"]),
)
filegroup(
name = "enc_headers",
srcs = glob(["enc/*.h"]),
)
filegroup(
name = "enc_sources",
srcs = glob(["enc/*.c"]),
)
cc_library(
name = "brotli_common",
srcs = [":common_sources"],
hdrs = [":common_headers"],
copts = STRICT_C_OPTIONS,
)
cc_library(
name = "brotli_dec",
srcs = [":dec_sources"],
hdrs = [":dec_headers"],
copts = STRICT_C_OPTIONS,
deps = [
":brotli_common",
],
)
cc_library(
name = "brotli_enc",
srcs = [":enc_sources"],
hdrs = [":enc_headers"],
copts = STRICT_C_OPTIONS,
deps = [
":brotli_common",
],
)
cc_binary(
name = "bro",
srcs = ["tools/bro.c"],
copts = STRICT_C_OPTIONS,
linkstatic = 1,
deps = [
":brotli_dec",
":brotli_enc",
],
)

View File

@@ -1,141 +0,0 @@
# Description:
# Brotli is a generic-purpose lossless compression algorithm.
package(
default_visibility = ["//visibility:public"],
)
licenses(["notice"]) # MIT
exports_files(["LICENSE"])
config_setting(
name = "clang-cl",
flag_values = {
"@bazel_tools//tools/cpp:compiler": "clang-cl",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "msvc",
flag_values = {
"@bazel_tools//tools/cpp:compiler": "msvc-cl",
},
visibility = ["//visibility:public"],
)
STRICT_C_OPTIONS = select({
":msvc": [],
":clang-cl": [
"/W4",
"-Wconversion",
"-Wlong-long",
"-Wmissing-declarations",
"-Wmissing-prototypes",
"-Wno-strict-aliasing",
"-Wshadow",
"-Wsign-compare",
"-Wno-sign-conversion",
],
"//conditions:default": [
"--pedantic-errors",
"-Wall",
"-Wconversion",
"-Werror",
"-Wextra",
"-Wlong-long",
"-Wmissing-declarations",
"-Wmissing-prototypes",
"-Wno-strict-aliasing",
"-Wshadow",
"-Wsign-compare",
],
})
filegroup(
name = "public_headers",
srcs = glob(["c/include/brotli/*.h"]),
)
filegroup(
name = "common_headers",
srcs = glob(["c/common/*.h"]),
)
filegroup(
name = "common_sources",
srcs = glob(["c/common/*.c"]),
)
filegroup(
name = "dec_headers",
srcs = glob(["c/dec/*.h"]),
)
filegroup(
name = "dec_sources",
srcs = glob(["c/dec/*.c"]),
)
filegroup(
name = "enc_headers",
srcs = glob(["c/enc/*.h"]),
)
filegroup(
name = "enc_sources",
srcs = glob(["c/enc/*.c"]),
)
cc_library(
name = "brotli_inc",
hdrs = [":public_headers"],
copts = STRICT_C_OPTIONS,
strip_include_prefix = "c/include",
)
cc_library(
name = "brotlicommon",
srcs = [":common_sources"],
hdrs = [":common_headers"],
copts = STRICT_C_OPTIONS,
deps = [":brotli_inc"],
)
cc_library(
name = "brotlidec",
srcs = [":dec_sources"],
hdrs = [":dec_headers"],
copts = STRICT_C_OPTIONS,
deps = [":brotlicommon"],
)
cc_library(
name = "brotlienc",
srcs = [":enc_sources"],
hdrs = [":enc_headers"],
copts = STRICT_C_OPTIONS,
linkopts = select({
":clang-cl": [],
":msvc": [],
"//conditions:default": ["-lm"],
}),
deps = [":brotlicommon"],
)
cc_binary(
name = "brotli",
srcs = ["c/tools/brotli.c"],
copts = STRICT_C_OPTIONS,
linkstatic = 1,
deps = [
":brotlidec",
":brotlienc",
],
)
filegroup(
name = "dictionary",
srcs = ["c/common/dictionary.bin"],
)

View File

@@ -1,291 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
## [1.2.0] - 2025-10-27
### SECURITY
- python: added `Decompressor::can_accept_more_data` method and optional
`output_buffer_limit` argument `Decompressor::process`;
that allows mitigation of unexpectedly large output;
reported by Charles Chan (https://github.com/charleswhchan)
### Added
- **decoder / encoder: added static initialization to reduce binary size**
- python: allow limiting decoder output (see SECURITY section)
- CLI: `brcat` alias; allow decoding concatenated brotli streams
- kt: pure Kotlin decoder
- cgo: support "raw" dictionaries
- build: Bazel modules
### Removed
- java: dropped `finalize()` for native entities
### Fixed
- java: in `compress` pass correct length to native encoder
### Improved
- build: install man pages
- build: updated / fixed / refined Bazel buildfiles
- encoder: faster encoding
- cgo: link via pkg-config
- python: modernize extension / allow multi-phase module initialization
### Changed
- decoder / encoder: static tables use "small" model (allows 2GiB+ binaries)
## [1.1.0] - 2023-08-28
### Added
- decoder: `BrotliDecoderAttachDictionary`
- decoder: `BrotliDecoderOnFinish` callback behind `BROTLI_REPORTING`
- decoder: `BrotliDecoderSetMetadataCallbacks`
- encoder: `BrotliEncoderPrepareDictionary`,
`BrotliEncoderDestroyPreparedDictionary`,
`BrotliEncoderAttachPreparedDictionary`
- decoder: `BrotliEncoderOnFinish` callback behind `BROTLI_REPORTING`
- common: `BrotliSharedDictionaryCreateInstance`,
`BrotliSharedDictionaryDestroyInstance`,
`BrotliSharedDictionaryAttach`
- CLI: `--dictionary` option
- java: encoder wrapper: `Parameters.mode`
- java: `Brotli{Input|Output}Stream.attachDictionary`
- java: wrapper: partial byte array input
- typescript: decoder (transpiled from Java)
### Removed
- build: `BROTLI_BUILD_PORTABLE` option
### Fixed
- java: JNI decoder failed sometimes on power of 2 payloads
### Improved
- java / js: smaller decoder footprint
- decoder: faster decoding
- encoder: faster encoding
- encoder: smaller stack frames
## [1.0.9] - 2020-08-27
Re-release of 1.0.8.
## [1.0.8] - 2020-08-27
### SECURITY
- CVE-2020-8927: potential overflow when input chunk is >2GiB
### Added
- encoder: `BROTLI_PARAM_STREAM_OFFSET`
### Improved
- CLI: better reporting
- CLI: workaround for "lying feof"
- java: faster decoding
- java: support "large window"
- encoder: use less memory
- release: filter sources for the tarball
## [1.0.7] - 2018-10-23
### Improved
- decoder: faster decoding on ARM CPU
## [1.0.6] - 2018-09-13
### Fixed
- build: AutoMake and CMake build
- java: JDK 8<->9 incompatibility
## [1.0.5] - 2018-06-27
### Added
- scripts: extraction of static dictionary from RFC
### Improved
- encoder: better compression at quality 1
- encoder: better compression with "large window"
## [1.0.4] - 2018-03-29
### Added
- encoder: `BROTLI_PARAM_NPOSTFIX`, `BROTLI_PARAM_NDIRECT`
- CLI: `--large_window` option
### Improved
- encoder: better compression
## [1.0.3] - 2018-03-02
### Added
- decoder: `BROTLI_DECODER_PARAM_LARGE_WINDOW` enum
- encoder: `BROTLI_PARAM_LARGE_WINDOW` enum
- java: `BrotliInputStream.setEager`
### Fixed
- build: AutoMake build in some environments
- encoder: fix one-shot q=10 1-byte input compression
### Improved
- encoder: better font compression
## [1.0.2] - 2017-11-28
### Added
- build: AutoMake
- research: better dictionary generators
## [1.0.1] - 2017-09-22
### Changed
- clarifications in `README.md`
## [1.0.0] - 2017-09-20
### Added
- decoder: `BrotliDecoderSetParameter`
- csharp: decoder (transpiled from Java)
- java: JNI wrappers
- javascript: decoder (transpiled from Java)
- python: streaming decompression
- research: dictionary generator
### Changed
- CLI: rename `bro` to `brotli`
### Removed
- decoder: `BrotliDecoderSetCustomDictionary`
- encoder: `BrotliEncoderSetCustomDictionary`
### Improved
- java: faster decoding
- encoder: faster compression
## [0.6.0] - 2017-04-10
### Added
- CLI: `--no-copy-stat option
- java: pure java decoder
- build: fuzzers
- research: `brotlidump` tool to explore brotli streams
- go: wrapper
### Removed
- decoder: API with plain `Brotli` prefix
### Deprecated
- encoder: `BrotliEncoderInputBlockSize`, `BrotliEncoderCopyInputToRingBuffer`,
`BrotliEncoderWriteData`
### Improved
- encoder: faster compression
- encoder: denser compression
- decoder: faster decompression
- python: release GIL
- python: use zero-copy API
## [0.5.2] - 2016-08-11
### Added
- common: `BROTLI_BOOL`, `BROTLI_TRUE`, `BROTLI_FALSE`
- decoder: API with `BrotliDecoder` prefix instead of plain `Brotli`
- build: Bazel, CMake
### Deprecated
- decoder: API with plain `Brotli` prefix
### Changed
- boolean argument / result types are re-branded as `BROTLI_BOOL`
### Improved
- build: reduced amount of warnings in various build environments
- encoder: faster compression
- encoder: lower memory usage
## [0.5.0] - 2016-06-15
### Added
- common: library has been assembled from shared parts of decoder and encoder
- encoder: C API
### Removed
- encoder: C++ API
## [0.4.0] - 2016-06-14
### Added
- encoder: faster compression modes (quality 0 and 1)
- decoder: `BrotliGetErrorCode`, `BrotliErrorString` and
`BROTLI_ERROR_CODES_LIST`
### Removed
- decoder: deprecated streaming API (using `BrotliInput`)
### Fixed
- decoder: possible pointer underflow
### Improved
- encoder: faster compression
## [0.3.0] - 2015-12-22
### LICENSE
License have been upgraded to more permissive MIT.
### Added
- CLI: `--window` option
- `tools/version.h` file
- decoder: low level streaming API
- decoder: custom memory manager API
### Deprecated
- decoder: streaming API using `BrotliInput` struct
### Fixed
- decoder: processing of uncompressed blocks
- encoder: possible division by zero
### Improved
- encoder: faster decompression
- build: more portable builds for various CPU architectures
## [0.2.0] - 2015-09-01
### Added
- CLI: `--verbose` and `--repeat` options
### Fixed
- decoder: processing of uncompressed blocks
- encoder: block stitching on quality 10 / 11
### Improved
- build: CI/CD integration
- build: better test coverage
- encoder: better compression of UTF-8 content
- encoder: faster decompression
## [0.1.0] - 2015-08-11
Initial release.

View File

@@ -1,169 +1,113 @@
# Available CMake versions:
# - Ubuntu 20.04 LTS : 3.16.3
# - Solaris 11.4 SRU 15 : 3.15
cmake_minimum_required(VERSION 3.15)
# Ubuntu 12.04 LTS has CMake 2.8.7, and is an important target since
# several CI services, such as Travis and Drone, use it. Solaris 11
# has 2.8.6, and it's not difficult to support if you already have to
# support 2.8.7.
cmake_minimum_required(VERSION 2.8.6)
# Since this project's version is loaded from other files, this policy
# will help suppress the warning generated by cmake.
# This policy is set because we can't provide "VERSION" in "project" command.
# Use `cmake --help-policy CMP0048` for more information.
cmake_policy(SET CMP0048 NEW)
project(brotli C)
if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "Setting build type to Release as none was specified")
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build" FORCE)
else()
message(STATUS "Build type is '${CMAKE_BUILD_TYPE}'")
endif()
include(CheckCSourceCompiles)
check_c_source_compiles(
"#if defined(__EMSCRIPTEN__)
int main() {return 0;}
#endif"
BROTLI_EMSCRIPTEN
)
if (BROTLI_EMSCRIPTEN)
message("-- Compiler is EMSCRIPTEN")
else()
message("-- Compiler is not EMSCRIPTEN")
endif()
if (BROTLI_EMSCRIPTEN)
message(STATUS "Switching to static build for EMSCRIPTEN")
set(BUILD_SHARED_LIBS OFF)
endif()
# Reflect CMake variable as a build option.
option(BUILD_SHARED_LIBS "Build shared libraries" ON)
set(BROTLI_BUILD_TOOLS ON CACHE BOOL "Build/install CLI tools")
set(BROTLI_BUILD_FOR_PACKAGE OFF CACHE BOOL "Build/install both shared and static libraries")
if (BROTLI_BUILD_FOR_PACKAGE AND NOT BUILD_SHARED_LIBS)
message(FATAL_ERROR "Both BROTLI_BUILD_FOR_PACKAGE and BUILD_SHARED_LIBS are set")
endif()
project(brotli)
# If Brotli is being bundled in another project, we don't want to
# install anything. However, we want to let people override this, so
# we'll use the BROTLI_BUNDLED_MODE variable to let them do that; just
# set it to OFF in your project before you add_subdirectory(brotli).
get_directory_property(BROTLI_PARENT_DIRECTORY PARENT_DIRECTORY)
if (NOT DEFINED BROTLI_BUNDLED_MODE)
if(BROTLI_BUNDLED_MODE STREQUAL "")
# Bundled mode hasn't been set one way or the other, set the default
# depending on whether or not we are the top-level project.
if (BROTLI_PARENT_DIRECTORY)
set(BROTLI_BUNDLED_MODE ON)
else()
if(BROTLI_PARENT_DIRECTORY)
set(BROTLI_BUNDLED_MODE OFF)
else()
set(BROTLI_BUNDLED_MODE ON)
endif()
endif() # BROTLI_BUNDLED_MODE
endif()
mark_as_advanced(BROTLI_BUNDLED_MODE)
include(GNUInstallDirs)
# Reads macro from .h file; it is expected to be a single-line define.
function(read_macro PATH MACRO OUTPUT)
file(STRINGS ${PATH} _line REGEX "^#define +${MACRO} +(.+)$")
string(REGEX REPLACE "^#define +${MACRO} +(.+)$" "\\1" _val "${_line}")
set(${OUTPUT} ${_val} PARENT_SCOPE)
endfunction(read_macro)
# Version information
read_macro("c/common/version.h" "BROTLI_VERSION_MAJOR" BROTLI_VERSION_MAJOR)
read_macro("c/common/version.h" "BROTLI_VERSION_MINOR" BROTLI_VERSION_MINOR)
read_macro("c/common/version.h" "BROTLI_VERSION_PATCH" BROTLI_VERSION_PATCH)
set(BROTLI_VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_PATCH}")
mark_as_advanced(BROTLI_VERSION BROTLI_VERSION_MAJOR BROTLI_VERSION_MINOR BROTLI_VERSION_PATCH)
# ABI Version information
read_macro("c/common/version.h" "BROTLI_ABI_CURRENT" BROTLI_ABI_CURRENT)
read_macro("c/common/version.h" "BROTLI_ABI_REVISION" BROTLI_ABI_REVISION)
read_macro("c/common/version.h" "BROTLI_ABI_AGE" BROTLI_ABI_AGE)
math(EXPR BROTLI_ABI_COMPATIBILITY "${BROTLI_ABI_CURRENT} - ${BROTLI_ABI_AGE}")
mark_as_advanced(BROTLI_ABI_CURRENT BROTLI_ABI_REVISION BROTLI_ABI_AGE BROTLI_ABI_COMPATIBILITY)
# Parse version information from tools/version.h. Normally we would
# define these values here and write them out to configuration file(s)
# (i.e., config.h), but in this case we parse them from
# tools/version.h to be less intrusive.
file(STRINGS "tools/version.h" BROTLI_VERSION REGEX "^#define BROTLI_VERSION \"+([0-9]+)\\.([0-9]+)\\.([0-9]+)\"")
string(REGEX REPLACE "^#define BROTLI_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\"$" "\\1" BROTLI_VERSION_MAJOR "${BROTLI_VERSION}")
string(REGEX REPLACE "^#define BROTLI_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\"$" "\\2" BROTLI_VERSION_MINOR "${BROTLI_VERSION}")
string(REGEX REPLACE "^#define BROTLI_VERSION \"([0-9]+)\\.([0-9]+)\\.([0-9]+)\"$" "\\3" BROTLI_VERSION_REVISION "${BROTLI_VERSION}")
mark_as_advanced(BROTLI_VERSION_MAJOR BROTLI_VERSION_MINOR BROTLI_VERSION_REVISION)
if (ENABLE_SANITIZER)
set(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
# By default, brotli depends on undefined behavior, but setting
# BROTLI_BUILD_PORTABLE should result in a build which does not.
if(ENABLE_SANITIZER STREQUAL "undefined")
add_definitions(-DBROTLI_BUILD_PORTABLE)
endif()
endif ()
include(CheckLibraryExists)
include(CheckFunctionExists)
set(LIBM_LIBRARY)
set(LIBM_DEP)
CHECK_LIBRARY_EXISTS(m log2 "" HAVE_LIB_M)
if (HAVE_LIB_M)
set(LIBM_LIBRARY "m")
if (NOT BUILD_SHARED_LIBS)
set(LIBM_DEP "-lm")
CHECK_FUNCTION_EXISTS(log2 LOG2_RES)
if(NOT LOG2_RES)
set(orig_req_libs "${CMAKE_REQUIRED_LIBRARIES}")
set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};m")
CHECK_FUNCTION_EXISTS(log2 LOG2_LIBM_RES)
if(LOG2_LIBM_RES)
set(LIBM_LIBRARY "m")
else()
message(FATAL_ERROR "log2() not found")
endif()
set(CMAKE_REQUIRED_LIBRARIES "${orig_req_libs}")
unset(LOG2_LIBM_RES)
unset(orig_req_libs)
endif()
unset(LOG2_RES)
set(BROTLI_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}")
set(BROTLI_LIBRARIES brotli_enc brotli_dec brotli_common ${LIBM_LIBRARY})
mark_as_advanced(BROTLI_INCLUDE_DIRS BROTLI_LIBRARIES)
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
add_definitions(-DOS_LINUX)
elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
add_definitions(-DOS_FREEBSD)
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
add_definitions(-DOS_MACOSX)
endif()
set(BROTLI_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/c/include")
mark_as_advanced(BROTLI_INCLUDE_DIRS)
add_library(brotli_common STATIC
common/dictionary.c)
add_library(brotli_dec STATIC
dec/bit_reader.c
dec/decode.c
dec/huffman.c
dec/state.c)
add_library(brotli_enc STATIC
enc/backward_references.c
enc/bit_cost.c
enc/block_splitter.c
enc/brotli_bit_stream.c
enc/cluster.c
enc/compress_fragment.c
enc/compress_fragment_two_pass.c
enc/encode.c
enc/entropy_encode.c
enc/histogram.c
enc/literal_cost.c
enc/memory.c
enc/metablock.c
enc/static_dict.c
enc/utf8_util.c)
if (BROTLI_BUILD_FOR_PACKAGE)
set(BROTLI_SHARED_LIBRARIES brotlienc brotlidec brotlicommon)
set(BROTLI_STATIC_LIBRARIES brotlienc-static brotlidec-static brotlicommon-static)
set(BROTLI_LIBRARIES ${BROTLI_SHARED_LIBRARIES} ${LIBM_LIBRARY})
else() # NOT BROTLI_BUILD_FOR_PACKAGE
if (BUILD_SHARED_LIBS)
set(BROTLI_SHARED_LIBRARIES brotlienc brotlidec brotlicommon)
set(BROTLI_STATIC_LIBRARIES)
else() # NOT BUILD_SHARED_LIBS
set(BROTLI_SHARED_LIBRARIES)
set(BROTLI_STATIC_LIBRARIES brotlienc brotlidec brotlicommon)
endif()
set(BROTLI_LIBRARIES ${BROTLI_SHARED_LIBRARIES} ${BROTLI_STATIC_LIBRARIES} ${LIBM_LIBRARY})
endif() # BROTLI_BUILD_FOR_PACKAGE
mark_as_advanced(BROTLI_LIBRARIES)
if (MSVC)
message(STATUS "Defining _CRT_SECURE_NO_WARNINGS to avoid warnings about security")
add_definitions(-D_CRT_SECURE_NO_WARNINGS)
endif()
file(GLOB_RECURSE BROTLI_COMMON_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} c/common/*.c)
file(GLOB_RECURSE BROTLI_DEC_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} c/dec/*.c)
file(GLOB_RECURSE BROTLI_ENC_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} c/enc/*.c)
add_library(brotlicommon ${BROTLI_COMMON_SOURCES})
add_library(brotlidec ${BROTLI_DEC_SOURCES})
add_library(brotlienc ${BROTLI_ENC_SOURCES})
if (BROTLI_BUILD_FOR_PACKAGE)
add_library(brotlicommon-static STATIC ${BROTLI_COMMON_SOURCES})
add_library(brotlidec-static STATIC ${BROTLI_DEC_SOURCES})
add_library(brotlienc-static STATIC ${BROTLI_ENC_SOURCES})
endif()
# Older CMake versions does not understand INCLUDE_DIRECTORIES property.
include_directories(${BROTLI_INCLUDE_DIRS})
if (BUILD_SHARED_LIBS)
foreach(lib ${BROTLI_SHARED_LIBRARIES})
target_compile_definitions(${lib} PUBLIC "BROTLI_SHARED_COMPILATION" )
string(TOUPPER "${lib}" LIB)
set_target_properties (${lib} PROPERTIES DEFINE_SYMBOL "${LIB}_SHARED_COMPILATION")
endforeach()
endif() # BUILD_SHARED_LIBS
foreach(lib ${BROTLI_SHARED_LIBRARIES} ${BROTLI_STATIC_LIBRARIES})
foreach(lib brotli_common brotli_dec brotli_enc)
target_link_libraries(${lib} ${LIBM_LIBRARY})
set_property(TARGET ${lib} APPEND PROPERTY INCLUDE_DIRECTORIES ${BROTLI_INCLUDE_DIRS})
set_target_properties(${lib} PROPERTIES
VERSION "${BROTLI_ABI_COMPATIBILITY}.${BROTLI_ABI_AGE}.${BROTLI_ABI_REVISION}"
SOVERSION "${BROTLI_ABI_COMPATIBILITY}")
if (NOT BROTLI_EMSCRIPTEN)
set_target_properties(${lib} PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
endif()
set_property(TARGET ${lib} APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${BROTLI_INCLUDE_DIRS}>")
endforeach() # BROTLI_xxx_LIBRARIES
VERSION ${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}
POSITION_INDEPENDENT_CODE TRUE)
target_link_libraries(brotlidec brotlicommon)
target_link_libraries(brotlienc brotlicommon)
set_property(TARGET ${lib} APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIRS}")
endforeach()
# For projects stuck on older versions of CMake, this will set the
# BROTLI_INCLUDE_DIRS and BROTLI_LIBRARIES variables so they still
@@ -171,86 +115,38 @@ target_link_libraries(brotlienc brotlicommon)
#
# include_directories(${BROTLI_INCLUDE_DIRS})
# target_link_libraries(foo ${BROTLI_LIBRARIES})
if (BROTLI_PARENT_DIRECTORY)
if(BROTLI_PARENT_DIRECTORY)
set(BROTLI_INCLUDE_DIRS "${BROTLI_INCLUDE_DIRS}" PARENT_SCOPE)
set(BROTLI_LIBRARIES "${BROTLI_LIBRARIES}" PARENT_SCOPE)
endif()
# Build the brotli executable
if (BROTLI_BUILD_TOOLS)
add_executable(brotli c/tools/brotli.c)
target_link_libraries(brotli ${BROTLI_LIBRARIES})
# brotli is a CLI tool
set_target_properties(brotli PROPERTIES MACOSX_BUNDLE OFF)
endif()
# Build the bro executable
add_executable(bro tools/bro.c)
target_link_libraries(bro ${BROTLI_LIBRARIES})
# Installation
if (NOT BROTLI_BUNDLED_MODE)
if (BROTLI_BUILD_TOOLS)
install(
TARGETS brotli
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
)
endif()
if(NOT BROTLI_BUNDLE_MODE)
include(GNUInstallDirs)
install(
TARGETS ${BROTLI_SHARED_LIBRARIES} ${BROTLI_STATIC_LIBRARIES}
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
)
install(
DIRECTORY ${BROTLI_INCLUDE_DIRS}/brotli
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
endif() # BROTLI_BUNDLED_MODE
install (TARGETS bro RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
endif()
# Tests
# Integration tests, those depend on `brotli` binary
if (NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
# If we're targeting Windows but not running on Windows, we need Wine
# to run the tests...
if (WIN32 AND NOT CMAKE_HOST_WIN32)
find_program(BROTLI_WRAPPER NAMES wine)
# If we're targeting Windows but not running on Windows, we need Wine
# to run the tests...
if(NOT BROTLI_DISABLE_TESTS)
if(WIN32 AND NOT CMAKE_HOST_WIN32)
find_program(BROTLI_WINE NAMES wine)
if (NOT BROTLI_WRAPPER)
if(NOT BROTLI_WINE)
message(STATUS "wine not found, disabling tests")
set(BROTLI_DISABLE_TESTS TRUE)
endif()
endif() # WIN32 emulation
if (BROTLI_EMSCRIPTEN)
find_program(BROTLI_WRAPPER NAMES node)
if (NOT BROTLI_WRAPPER)
message(STATUS "node not found, disabling tests")
set(BROTLI_DISABLE_TESTS TRUE)
endif()
endif() # BROTLI_EMSCRIPTEN
endif() # BROTLI_DISABLE_TESTS
# NB: BROTLI_DISABLE_TESTS might have changed.
if (NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
# If our compiler is a cross-compiler that we know about (arm/aarch64),
# then we need to use qemu to execute the tests.
if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabihf-.*$")
message(STATUS "Detected arm-linux-gnueabihf cross-compilation")
set(BROTLI_WRAPPER "qemu-arm")
set(BROTLI_WRAPPER_LD_PREFIX "/usr/arm-linux-gnueabihf")
endif()
if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabi-.*$")
message(STATUS "Detected arm-linux-gnueabi cross-compilation")
set(BROTLI_WRAPPER "qemu-arm")
set(BROTLI_WRAPPER_LD_PREFIX "/usr/arm-linux-gnueabi")
endif()
if ("${CMAKE_C_COMPILER}" MATCHES "^.*/aarch64-linux-gnu-.*$")
message(STATUS "Detected aarch64-linux-gnu cross-compilation")
set(BROTLI_WRAPPER "qemu-aarch64")
set(BROTLI_WRAPPER_LD_PREFIX "/usr/aarch64-linux-gnu")
endif()
endif()
if(NOT BROTLI_DISABLE_TESTS)
include(CTest)
enable_testing()
@@ -259,9 +155,9 @@ if (NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
tests/testdata/asyoulik.txt
tests/testdata/lcet10.txt
tests/testdata/plrabn12.txt
c/enc/encode.c
c/common/dictionary.h
c/dec/decode.c)
enc/encode.c
common/dictionary.h
dec/decode.c)
foreach(INPUT ${ROUNDTRIP_INPUTS})
get_filename_component(OUTPUT_NAME "${INPUT}" NAME)
@@ -269,21 +165,16 @@ if (NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
set(OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME}")
set(INPUT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}")
if (EXISTS "${INPUT_FILE}")
foreach(quality 1 6 9 11)
add_test(NAME "${BROTLI_TEST_PREFIX}roundtrip/${INPUT}/${quality}"
COMMAND "${CMAKE_COMMAND}"
-DBROTLI_WRAPPER=${BROTLI_WRAPPER}
-DBROTLI_WRAPPER_LD_PREFIX=${BROTLI_WRAPPER_LD_PREFIX}
-DBROTLI_CLI=$<TARGET_FILE:brotli>
-DQUALITY=${quality}
-DINPUT=${INPUT_FILE}
-DOUTPUT=${OUTPUT_FILE}.${quality}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-roundtrip-test.cmake)
endforeach()
else()
message(NOTICE "Test file ${INPUT} does not exist; OK on tarball builds; consider running scripts/download_testdata.sh before configuring.")
endif()
foreach(quality 1 6 9 11)
add_test(NAME "${BROTLI_TEST_PREFIX}roundtrip/${INPUT}/${quality}"
COMMAND "${CMAKE_COMMAND}"
-DBROTLI_WRAPPER=${BROTLI_WINE}
-DBROTLI_CLI=$<TARGET_FILE:bro>
-DQUALITY=${quality}
-DINPUT=${INPUT_FILE}
-DOUTPUT=${OUTPUT_FILE}.${quality}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-roundtrip-test.cmake)
endforeach()
endforeach()
file(GLOB_RECURSE
@@ -292,106 +183,11 @@ if (NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
tests/testdata/*.compressed*)
foreach(INPUT ${COMPATIBILITY_INPUTS})
string(REGEX REPLACE "([a-zA-Z0-9\\.]+)\\.compressed(\\.[0-9]+)?$" "\\1" UNCOMPRESSED_INPUT "${INPUT}")
if (EXISTS ${UNCOMPRESSED_INPUT})
add_test(NAME "${BROTLI_TEST_PREFIX}compatibility/${INPUT}"
COMMAND "${CMAKE_COMMAND}"
-DBROTLI_WRAPPER=${BROTLI_WRAPPER}
-DBROTLI_WRAPPER_LD_PREFIX=${BROTLI_WRAPPER_LD_PREFIX}
-DBROTLI_CLI=$<TARGET_FILE:brotli>
-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake)
endif()
add_test(NAME "${BROTLI_TEST_PREFIX}compatibility/${INPUT}"
COMMAND "${CMAKE_COMMAND}"
-DBROTLI_WRAPPER=${BROTLI_WINE}
-DBROTLI_CLI=$<TARGET_FILE:bro>
-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake)
endforeach()
endif() # BROTLI_DISABLE_TESTS
# Generate a pkg-config files
function(generate_pkg_config_path outvar path)
string(LENGTH "${path}" path_length)
set(path_args ${ARGV})
list(REMOVE_AT path_args 0 1)
list(LENGTH path_args path_args_remaining)
set("${outvar}" "${path}")
while(path_args_remaining GREATER 1)
list(GET path_args 0 name)
list(GET path_args 1 value)
get_filename_component(value_full "${value}" ABSOLUTE)
string(LENGTH "${value}" value_length)
if (path_length EQUAL value_length AND path STREQUAL value)
set("${outvar}" "\${${name}}")
break()
elseif (path_length GREATER value_length)
# We might be in a subdirectory of the value, but we have to be
# careful about a prefix matching but not being a subdirectory
# (for example, /usr/lib64 is not a subdirectory of /usr/lib).
# We'll do this by making sure the next character is a directory
# separator.
string(SUBSTRING "${path}" ${value_length} 1 sep)
if (sep STREQUAL "/")
string(SUBSTRING "${path}" 0 ${value_length} s)
if (s STREQUAL value)
string(SUBSTRING "${path}" "${value_length}" -1 suffix)
set("${outvar}" "\${${name}}${suffix}")
break()
endif()
endif()
endif()
list(REMOVE_AT path_args 0 1)
list(LENGTH path_args path_args_remaining)
endwhile()
set("${outvar}" "${${outvar}}" PARENT_SCOPE)
endfunction(generate_pkg_config_path)
function(transform_pc_file INPUT_FILE OUTPUT_FILE VERSION)
file(READ ${INPUT_FILE} TEXT)
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
string(REGEX REPLACE "@prefix@" "${PREFIX}" TEXT ${TEXT})
string(REGEX REPLACE "@exec_prefix@" "${PREFIX}" TEXT ${TEXT})
string(REGEX REPLACE "@libm@" "${LIBM_DEP}" TEXT ${TEXT})
generate_pkg_config_path(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}" prefix "${PREFIX}")
string(REGEX REPLACE "@libdir@" "${LIBDIR}" TEXT ${TEXT})
generate_pkg_config_path(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}" prefix "${PREFIX}")
string(REGEX REPLACE "@includedir@" "${INCLUDEDIR}" TEXT ${TEXT})
string(REGEX REPLACE "@PACKAGE_VERSION@" "${VERSION}" TEXT ${TEXT})
file(WRITE ${OUTPUT_FILE} ${TEXT})
endfunction()
transform_pc_file("scripts/libbrotlicommon.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlicommon.pc" "${BROTLI_VERSION}")
transform_pc_file("scripts/libbrotlidec.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlidec.pc" "${BROTLI_VERSION}")
transform_pc_file("scripts/libbrotlienc.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlienc.pc" "${BROTLI_VERSION}")
if (NOT BROTLI_BUNDLED_MODE)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libbrotlicommon.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libbrotlidec.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libbrotlienc.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
endif() # BROTLI_BUNDLED_MODE
if (BROTLI_BUILD_TOOLS)
install(FILES "docs/brotli.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
endif()
install(FILES docs/constants.h.3 docs/decode.h.3 docs/encode.h.3 docs/types.h.3
DESTINATION "${CMAKE_INSTALL_MANDIR}/man3")
if (ENABLE_COVERAGE STREQUAL "yes")
setup_target_for_coverage(coverage test coverage)
endif()

View File

@@ -21,11 +21,6 @@ frustration later on.
All submissions, including submissions by project members, require review. We
use Github pull requests for this purpose.
### Code style
Code should follow applicable formatting and style guides described in
[Google Style Guides](https://google.github.io/styleguide/). C code should be
C89 compatible.
### The small print
Contributions made by corporations are covered by a different agreement than
the one above, the [Software Grant and Corporate Contributor License Agreement]

View File

@@ -1,18 +1,16 @@
include CONTRIBUTING.md
include c/common/*.c
include c/common/*.h
include c/dec/*.c
include c/dec/*.h
include c/enc/*.c
include c/enc/*.h
include c/include/brotli/*.h
include common/*.c
include common/*.h
include dec/*.c
include dec/*.h
include enc/*.c
include enc/*.h
include LICENSE
include MANIFEST.in
include python/_brotli.cc
include python/brotli.py
include python/bro.py
include python/brotlimodule.cc
include python/README.md
include python/tests/*
include README.md
include setup.py
include tests/testdata/*
include c/tools/brotli.c
include tools/bro.c
include tools/version.h

View File

@@ -1,12 +0,0 @@
# Copyright 2025 The Brotli Authors. All rights reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
"""Brotli reference implementation"""
module(
name = "brotli",
version = "1.2.0",
repo_name = "org_brotli",
)

41
Makefile Normal file
View File

@@ -0,0 +1,41 @@
OS := $(shell uname)
LIBSOURCES = $(wildcard common/*.c) $(wildcard dec/*.c) $(wildcard enc/*.c)
SOURCES = $(LIBSOURCES) tools/bro.c
BINDIR = bin
OBJDIR = $(BINDIR)/obj
LIBOBJECTS = $(addprefix $(OBJDIR)/, $(LIBSOURCES:.c=.o))
OBJECTS = $(addprefix $(OBJDIR)/, $(SOURCES:.c=.o))
LIB_A = libbrotli.a
EXECUTABLE = bro
DIRS = $(OBJDIR)/common $(OBJDIR)/dec $(OBJDIR)/enc \
$(OBJDIR)/tools $(BINDIR)/tmp
CFLAGS += -O2
ifeq ($(os), Darwin)
CPPFLAGS += -DOS_MACOSX
endif
all: test
@:
.PHONY: all clean test
$(DIRS):
mkdir -p $@
$(OBJECTS): $(DIRS)
$(CC) $(CFLAGS) $(CPPFLAGS) \
-c $(patsubst %.o,%.c,$(patsubst $(OBJDIR)/%,%,$@)) -o $@
$(EXECUTABLE): $(OBJECTS)
$(CC) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE)
lib: $(LIBOBJECTS)
rm -f $(LIB_A)
ar -crs $(LIB_A) $(LIBOBJECTS)
test: $(EXECUTABLE)
tests/compatibility_test.sh
tests/roundtrip_test.sh
clean:
rm -rf $(BINDIR) $(LIB_A)

15
README
View File

@@ -1,15 +0,0 @@
BROTLI DATA COMPRESSION LIBRARY
Brotli is a generic-purpose lossless compression algorithm that compresses data
using a combination of a modern variant of the LZ77 algorithm, Huffman coding
and 2nd order context modeling, with a compression ratio comparable to the best
currently available general-purpose compression methods. It is similar in speed
with deflate but offers more dense compression.
The specification of the Brotli Compressed Data Format is defined in RFC 7932
https://datatracker.ietf.org/doc/html/rfc7932
Brotli is open-sourced under the MIT License, see the LICENSE file.
Brotli mailing list:
https://groups.google.com/g/brotli

102
README.md
View File

@@ -1,9 +1,3 @@
<p align="center">
<img src="https://github.com/google/brotli/actions/workflows/build_test.yml/badge.svg" alt="GitHub Actions Build Status" href="https://github.com/google/brotli/actions?query=branch%3Amaster">
<img src="https://oss-fuzz-build-logs.storage.googleapis.com/badges/brotli.svg" alt="Fuzzing Status" href="https://oss-fuzz-build-logs.storage.googleapis.com/index.html#brotli">
</p>
<p align="center"><img src="https://brotli.org/brotli.svg" alt="Brotli" width="64"></p>
### Introduction
Brotli is a generic-purpose lossless compression algorithm that compresses data
@@ -12,103 +6,21 @@ and 2nd order context modeling, with a compression ratio comparable to the best
currently available general-purpose compression methods. It is similar in speed
with deflate but offers more dense compression.
The specification of the Brotli Compressed Data Format is defined in
[RFC 7932](https://datatracker.ietf.org/doc/html/rfc7932).
The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://www.ietf.org/rfc/rfc7932.txt).
Brotli is open-sourced under the MIT License, see the LICENSE file.
> **Please note:** brotli is a "stream" format; it does not contain
> meta-information, like checksums or uncompressed data length. It is possible
> to modify "raw" ranges of the compressed stream and the decoder will not
> notice that.
Brotli mailing list:
https://groups.google.com/forum/#!forum/brotli
### Installation
In most Linux distributions, installing `brotli` is just a matter of using
the package management system. For example in Debian-based distributions:
`apt install brotli` will install `brotli`. On MacOS, you can use
[Homebrew](https://brew.sh/): `brew install brotli`.
[![brotli packaging status](https://repology.org/badge/vertical-allrepos/brotli.svg?exclude_unsupported=1&columns=3&exclude_sources=modules,site&header=brotli%20packaging%20status)](https://repology.org/project/brotli/versions)
Of course you can also build brotli from sources.
### Build instructions
#### Vcpkg
You can download and install brotli using the
[vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
./bootstrap-vcpkg.sh
./vcpkg integrate install
./vcpkg install brotli
The brotli port in vcpkg is kept up to date by Microsoft team members and
community contributors. If the version is out of date, please [create an issue
or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
#### Bazel
See [Bazel](https://www.bazel.build/)
#### CMake
The basic commands to build and install brotli are:
$ mkdir out && cd out
$ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed ..
$ cmake --build . --config Release --target install
You can use other [CMake](https://cmake.org/) configuration.
#### Python
To install the latest release of the Python module, run the following:
$ pip install brotli
To install the tip-of-the-tree version, run:
$ pip install --upgrade git+https://github.com/google/brotli
See the [Python readme](python/README.md) for more details on installing
from source, development, and testing.
### Contributing
We glad to answer/library related questions in
[brotli mailing list](https://groups.google.com/g/brotli).
Regular issues / feature requests should be reported in
[issue tracker](https://github.com/google/brotli/issues).
For reporting vulnerability please read [SECURITY](SECURITY.md).
For contributing changes please read [CONTRIBUTING](CONTRIBUTING.md).
[![Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
### Benchmarks
* [Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/) / [Unstable Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/unstable/)
* [Large Text Compression Benchmark](https://mattmahoney.net/dc/text.html)
* [Large Text Compression Benchmark](http://mattmahoney.net/dc/text.html)
* [Lzturbo Benchmark](https://sites.google.com/site/powturbo/home/benchmark)
### Related projects
> **Disclaimer:** Brotli authors take no responsibility for the third party projects mentioned in this section.
Independent [decoder](https://github.com/madler/brotli) implementation by Mark Adler, based entirely on format specification.
Independent [decoder](https://github.com/madler/brotli) implementation
by Mark Adler, based entirely on format specification.
JavaScript port of brotli [decoder](https://github.com/devongovett/brotli.js).
Could be used directly via `npm install brotli`
Hand ported [decoder / encoder](https://github.com/dominikhlbg/BrotliHaxe)
in haxe by Dominik Homberger.
Output source code: JavaScript, PHP, Python, Java and C#
7Zip [plugin](https://github.com/mcmilk/7-Zip-Zstd)
Dart compression framework with
[fast FFI-based Brotli implementation](https://pub.dev/documentation/es_compression/latest/brotli/)
with ready-to-use prebuilt binaries for Win/Linux/Mac
JavaScript port of brotli [decoder](https://github.com/devongovett/brotli.js). Could be used directly via `npm install brotli`

View File

@@ -1,6 +0,0 @@
### Reporting
To report a security issue, please use [https://g.co/vulnz](https://g.co/vulnz).
We use g.co/vulnz for our intake, and do coordination and disclosure here on
GitHub (including using GitHub Security Advisory). The Google Security Team will
respond within 5 working days of your report on g.co/vulnz.

4
WORKSPACE Normal file
View File

@@ -0,0 +1,4 @@
# Description:
# Bazel workspace file for Brotli.
workspace(name = "io_brotli")

114
appveyor.yml Normal file
View File

@@ -0,0 +1,114 @@
environment:
global:
# SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
# /E:ON and /V:ON options are not enabled in the batch script intepreter
# See: http://stackoverflow.com/a/13751649/163740
WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd"
matrix:
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 14 2015 Win64
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 14 2015
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 12 2013 Win64
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 12 2013
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python27"
PYTHON_VERSION: "2.7.x"
PYTHON_ARCH: "32"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python34"
PYTHON_VERSION: "3.4.x"
PYTHON_ARCH: "32"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python35"
PYTHON_VERSION: "3.5.0"
PYTHON_ARCH: "32"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python27-x64"
PYTHON_VERSION: "2.7.x"
PYTHON_ARCH: "64"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python34-x64"
PYTHON_VERSION: "3.4.x"
PYTHON_ARCH: "64"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python35-x64"
PYTHON_VERSION: "3.5.0"
PYTHON_ARCH: "64"
# init:
# - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
install:
- ps: |
if ($env:BUILD_SYSTEM -eq "Python") {
# install Python and pip when not already installed
if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
# prepend newly installed Python to the PATH
$env:Path = $env:PYTHON + ';' + $env:PYTHON + '\\Scripts;' + $env:PATH
# upgrade pip to avoid out-of-date warnings
pip install --disable-pip-version-check --user --upgrade pip
# install/upgrade setuptools and wheel to build packages
pip install --upgrade setuptools wheel
}
before_build:
- ps: |
if ($env:BUILD_SYSTEM -eq "CMake") {
mkdir builddir
cd builddir
cmake -G "$env:GENERATOR" ..
}
build_script:
- ps: |
if ($env:BUILD_SYSTEM -eq "CMake") {
cmake --build . --config Debug
}
test_script:
- ps: |
if ($env:BUILD_SYSTEM -eq "CMake") {
ctest --output-on-failure --interactive-debug-mode 0 -C Debug
} else {
python setup.py build_ext test
}
after_test:
- if "%BUILD_SYSTEM%" == "Python" ( pip wheel -w dist .)
- if "%BUILD_SYSTEM%" == "Python" ( python setup.py sdist --formats=gztar,zip )
artifacts:
# archive the generated packages in the ci.appveyor.com build report
- path: dist\*.whl
- path: dist\*.zip
- path: dist\*.tar.gz
# For info, see: http://www.appveyor.com/docs/deployment/github
deploy:
- provider: GitHub
auth_token:
secure: dfL56DgbwuGJNNE5GzKi/pAgBQnJ37Du+AnCtnsTnIYxpis8ah3fPmA/G+bn4NJ3
artifact:
draft: false
prerelease: false
on:
appveyor_repo_tag: true

177
appveyor/install.ps1 Normal file
View File

@@ -0,0 +1,177 @@
# Sample script to install Python and pip under Windows
# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
# Source: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/install.ps1
$BASE_URL = "https://www.python.org/ftp/python/"
$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
$GET_PIP_PATH = "C:\get-pip.py"
$PYTHON_PRERELEASE_REGEX = @"
(?x)
(?<major>\d+)
\.
(?<minor>\d+)
\.
(?<micro>\d+)
(?<prerelease>[a-z]{1,2}\d+)
"@
function Download ($filename, $url) {
$webclient = New-Object System.Net.WebClient
$basedir = $pwd.Path + "\"
$filepath = $basedir + $filename
if (Test-Path $filename) {
Write-Host "Reusing" $filepath
return $filepath
}
# Download and retry up to 3 times in case of network transient errors.
Write-Host "Downloading" $filename "from" $url
$retry_attempts = 2
for ($i = 0; $i -lt $retry_attempts; $i++) {
try {
$webclient.DownloadFile($url, $filepath)
break
}
Catch [Exception]{
Start-Sleep 1
}
}
if (Test-Path $filepath) {
Write-Host "File saved at" $filepath
} else {
# Retry once to get the error message if any at the last try
$webclient.DownloadFile($url, $filepath)
}
return $filepath
}
function ParsePythonVersion ($python_version) {
if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
$matches.prerelease)
}
$version_obj = [version]$python_version
return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
}
function DownloadPython ($python_version, $platform_suffix) {
$major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
if (($major -le 2 -and $micro -eq 0) `
-or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
) {
$dir = "$major.$minor"
$python_version = "$major.$minor$prerelease"
} else {
$dir = "$major.$minor.$micro"
}
if ($prerelease) {
if (($major -le 2) `
-or ($major -eq 3 -and $minor -eq 1) `
-or ($major -eq 3 -and $minor -eq 2) `
-or ($major -eq 3 -and $minor -eq 3) `
) {
$dir = "$dir/prev"
}
}
if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
$ext = "msi"
if ($platform_suffix) {
$platform_suffix = ".$platform_suffix"
}
} else {
$ext = "exe"
if ($platform_suffix) {
$platform_suffix = "-$platform_suffix"
}
}
$filename = "python-$python_version$platform_suffix.$ext"
$url = "$BASE_URL$dir/$filename"
$filepath = Download $filename $url
return $filepath
}
function InstallPython ($python_version, $architecture, $python_home) {
Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
if (Test-Path $python_home) {
Write-Host $python_home "already exists, skipping."
return $false
}
if ($architecture -eq "32") {
$platform_suffix = ""
} else {
$platform_suffix = "amd64"
}
$installer_path = DownloadPython $python_version $platform_suffix
$installer_ext = [System.IO.Path]::GetExtension($installer_path)
Write-Host "Installing $installer_path to $python_home"
$install_log = $python_home + ".log"
if ($installer_ext -eq '.msi') {
InstallPythonMSI $installer_path $python_home $install_log
} else {
InstallPythonEXE $installer_path $python_home $install_log
}
if (Test-Path $python_home) {
Write-Host "Python $python_version ($architecture) installation complete"
} else {
Write-Host "Failed to install Python in $python_home"
Get-Content -Path $install_log
Exit 1
}
}
function InstallPythonEXE ($exepath, $python_home, $install_log) {
$install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
RunCommand $exepath $install_args
}
function InstallPythonMSI ($msipath, $python_home, $install_log) {
$install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
$uninstall_args = "/qn /x $msipath"
RunCommand "msiexec.exe" $install_args
if (-not(Test-Path $python_home)) {
Write-Host "Python seems to be installed else-where, reinstalling."
RunCommand "msiexec.exe" $uninstall_args
RunCommand "msiexec.exe" $install_args
}
}
function RunCommand ($command, $command_args) {
Write-Host $command $command_args
Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
}
function InstallPip ($python_home) {
$pip_path = $python_home + "\Scripts\pip.exe"
$python_path = $python_home + "\python.exe"
if (-not(Test-Path $pip_path)) {
Write-Host "Installing pip..."
$webclient = New-Object System.Net.WebClient
$webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
Write-Host "Executing:" $python_path $GET_PIP_PATH
& $python_path $GET_PIP_PATH
} else {
Write-Host "pip already installed."
}
}
function main () {
InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
InstallPip $env:PYTHON
}
main

View File

@@ -0,0 +1,80 @@
:: To build extensions for 64 bit Python 3, we need to configure environment
:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
::
:: To build extensions for 64 bit Python 2, we need to configure environment
:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
::
:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
:: environment configurations.
::
:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
:: cmd interpreter, at least for (SDK v7.0)
::
:: More details at:
:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
:: http://stackoverflow.com/a/13751649/163740
::
:: Original source:
:: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/run_with_env.cmd
::
:: Author: Olivier Grisel
:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
@ECHO OFF
SET COMMAND_TO_RUN=%*
SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
:: Extract the major and minor versions, and allow for the minor version to be
:: more than 9. This requires the version number to have two dots in it.
SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
IF "%PYTHON_VERSION:~3,1%" == "." (
SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
) ELSE (
SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
)
:: Based on the Python version, determine what SDK version to use, and whether
:: to set the SDK for 64-bit.
IF %MAJOR_PYTHON_VERSION% == 2 (
SET WINDOWS_SDK_VERSION="v7.0"
SET SET_SDK_64=Y
) ELSE (
IF %MAJOR_PYTHON_VERSION% == 3 (
SET WINDOWS_SDK_VERSION="v7.1"
IF %MINOR_PYTHON_VERSION% LEQ 4 (
SET SET_SDK_64=Y
) ELSE (
SET SET_SDK_64=N
IF EXIST "%WIN_WDK%" (
:: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
REN "%WIN_WDK%" 0wdf
)
)
) ELSE (
ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
EXIT 1
)
)
IF %PYTHON_ARCH% == 64 (
IF %SET_SDK_64% == Y (
ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
SET DISTUTILS_USE_SDK=1
SET MSSdk=1
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT 1
) ELSE (
ECHO Using default MSVC build environment for 64 bit architecture
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT 1
)
) ELSE (
ECHO Using default MSVC build environment for 32 bit architecture
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT 1
)

View File

@@ -1,15 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "constants.h"
const BROTLI_MODEL("small")
BrotliPrefixCodeRange _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
{1, 2}, {5, 2}, {9, 2}, {13, 2}, {17, 3}, {25, 3},
{33, 3}, {41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4},
{113, 5}, {145, 5}, {177, 5}, {209, 5}, {241, 6}, {305, 6},
{369, 7}, {497, 8}, {753, 9}, {1265, 10}, {2289, 11}, {4337, 12},
{8433, 13}, {16625, 24}};

View File

@@ -1,198 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/**
* @file
* Common constants used in decoder and encoder API.
*/
#ifndef BROTLI_COMMON_CONSTANTS_H_
#define BROTLI_COMMON_CONSTANTS_H_
#include "platform.h"
/* Specification: 7.3. Encoding of the context map */
#define BROTLI_CONTEXT_MAP_MAX_RLE 16
/* Specification: 2. Compressed representation overview */
#define BROTLI_MAX_NUMBER_OF_BLOCK_TYPES 256
/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
#define BROTLI_NUM_LITERAL_SYMBOLS 256
#define BROTLI_NUM_COMMAND_SYMBOLS 704
#define BROTLI_NUM_BLOCK_LEN_SYMBOLS 26
#define BROTLI_MAX_CONTEXT_MAP_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + \
BROTLI_CONTEXT_MAP_MAX_RLE)
#define BROTLI_MAX_BLOCK_TYPE_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 2)
/* Specification: 3.5. Complex prefix codes */
#define BROTLI_REPEAT_PREVIOUS_CODE_LENGTH 16
#define BROTLI_REPEAT_ZERO_CODE_LENGTH 17
#define BROTLI_CODE_LENGTH_CODES (BROTLI_REPEAT_ZERO_CODE_LENGTH + 1)
/* "code length of 8 is repeated" */
#define BROTLI_INITIAL_REPEATED_CODE_LENGTH 8
/* "Large Window Brotli" */
/**
* The theoretical maximum number of distance bits specified for large window
* brotli, for 64-bit encoders and decoders. Even when in practice 32-bit
* encoders and decoders only support up to 30 max distance bits, the value is
* set to 62 because it affects the large window brotli file format.
* Specifically, it affects the encoding of simple huffman tree for distances,
* see Specification RFC 7932 chapter 3.4.
*/
#define BROTLI_LARGE_MAX_DISTANCE_BITS 62U
#define BROTLI_LARGE_MIN_WBITS 10
/**
* The maximum supported large brotli window bits by the encoder and decoder.
* Large window brotli allows up to 62 bits, however the current encoder and
* decoder, designed for 32-bit integers, only support up to 30 bits maximum.
*/
#define BROTLI_LARGE_MAX_WBITS 30
/* Specification: 4. Encoding of distances */
#define BROTLI_NUM_DISTANCE_SHORT_CODES 16
/**
* Maximal number of "postfix" bits.
*
* Number of "postfix" bits is stored as 2 bits in meta-block header.
*/
#define BROTLI_MAX_NPOSTFIX 3
#define BROTLI_MAX_NDIRECT 120
#define BROTLI_MAX_DISTANCE_BITS 24U
#define BROTLI_DISTANCE_ALPHABET_SIZE(NPOSTFIX, NDIRECT, MAXNBITS) ( \
BROTLI_NUM_DISTANCE_SHORT_CODES + (NDIRECT) + \
((MAXNBITS) << ((NPOSTFIX) + 1)))
/* BROTLI_NUM_DISTANCE_SYMBOLS == 1128 */
#define BROTLI_NUM_DISTANCE_SYMBOLS \
BROTLI_DISTANCE_ALPHABET_SIZE( \
BROTLI_MAX_NDIRECT, BROTLI_MAX_NPOSTFIX, BROTLI_LARGE_MAX_DISTANCE_BITS)
/* ((1 << 26) - 4) is the maximal distance that can be expressed in RFC 7932
brotli stream using NPOSTFIX = 0 and NDIRECT = 0. With other NPOSTFIX and
NDIRECT values distances up to ((1 << 29) + 88) could be expressed. */
#define BROTLI_MAX_DISTANCE 0x3FFFFFC
/* ((1 << 31) - 4) is the safe distance limit. Using this number as a limit
allows safe distance calculation without overflows, given the distance
alphabet size is limited to corresponding size
(see kLargeWindowDistanceCodeLimits). */
#define BROTLI_MAX_ALLOWED_DISTANCE 0x7FFFFFFC
/* Specification: 4. Encoding of Literal Insertion Lengths and Copy Lengths */
#define BROTLI_NUM_INS_COPY_CODES 24
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */
#define BROTLI_LITERAL_CONTEXT_BITS 6
/* 7.2. Context ID for distances */
#define BROTLI_DISTANCE_CONTEXT_BITS 2
/* 9.1. Format of the Stream Header */
/* Number of slack bytes for window size. Don't confuse
with BROTLI_NUM_DISTANCE_SHORT_CODES. */
#define BROTLI_WINDOW_GAP 16
#define BROTLI_MAX_BACKWARD_LIMIT(W) (((size_t)1 << (W)) - BROTLI_WINDOW_GAP)
typedef struct BrotliDistanceCodeLimit {
uint32_t max_alphabet_size;
uint32_t max_distance;
} BrotliDistanceCodeLimit;
/* This function calculates maximal size of distance alphabet, such that the
distances greater than the given values can not be represented.
This limits are designed to support fast and safe 32-bit decoders.
"32-bit" means that signed integer values up to ((1 << 31) - 1) could be
safely expressed.
Brotli distance alphabet symbols do not represent consecutive distance
ranges. Each distance alphabet symbol (excluding direct distances and short
codes), represent interleaved (for NPOSTFIX > 0) range of distances.
A "group" of consecutive (1 << NPOSTFIX) symbols represent non-interleaved
range. Two consecutive groups require the same amount of "extra bits".
It is important that distance alphabet represents complete "groups".
To avoid complex logic on encoder side about interleaved ranges
it was decided to restrict both sides to complete distance code "groups".
*/
BROTLI_UNUSED_FUNCTION BrotliDistanceCodeLimit BrotliCalculateDistanceCodeLimit(
uint32_t max_distance, uint32_t npostfix, uint32_t ndirect) {
BrotliDistanceCodeLimit result;
/* Marking this function as unused, because not all files
including "constants.h" use it -> compiler warns about that. */
BROTLI_UNUSED(&BrotliCalculateDistanceCodeLimit);
if (max_distance <= ndirect) {
/* This case never happens / exists only for the sake of completeness. */
result.max_alphabet_size = max_distance + BROTLI_NUM_DISTANCE_SHORT_CODES;
result.max_distance = max_distance;
return result;
} else {
/* The first prohibited value. */
uint32_t forbidden_distance = max_distance + 1;
/* Subtract "directly" encoded region. */
uint32_t offset = forbidden_distance - ndirect - 1;
uint32_t ndistbits = 0;
uint32_t tmp;
uint32_t half;
uint32_t group;
/* Postfix for the last dcode in the group. */
uint32_t postfix = (1u << npostfix) - 1;
uint32_t extra;
uint32_t start;
/* Remove postfix and "head-start". */
offset = (offset >> npostfix) + 4;
/* Calculate the number of distance bits. */
tmp = offset / 2;
/* Poor-man's log2floor, to avoid extra dependencies. */
while (tmp != 0) {ndistbits++; tmp = tmp >> 1;}
/* One bit is covered with subrange addressing ("half"). */
ndistbits--;
/* Find subrange. */
half = (offset >> ndistbits) & 1;
/* Calculate the "group" part of dcode. */
group = ((ndistbits - 1) << 1) | half;
/* Calculated "group" covers the prohibited distance value. */
if (group == 0) {
/* This case is added for correctness; does not occur for limit > 128. */
result.max_alphabet_size = ndirect + BROTLI_NUM_DISTANCE_SHORT_CODES;
result.max_distance = ndirect;
return result;
}
/* Decrement "group", so it is the last permitted "group". */
group--;
/* After group was decremented, ndistbits and half must be recalculated. */
ndistbits = (group >> 1) + 1;
/* The last available distance in the subrange has all extra bits set. */
extra = (1u << ndistbits) - 1;
/* Calculate region start. NB: ndistbits >= 1. */
start = (1u << (ndistbits + 1)) - 4;
/* Move to subregion. */
start += (group & 1) << ndistbits;
/* Calculate the alphabet size. */
result.max_alphabet_size = ((group << npostfix) | postfix) + ndirect +
BROTLI_NUM_DISTANCE_SHORT_CODES + 1;
/* Calculate the maximal distance representable by alphabet. */
result.max_distance = ((start + extra) << npostfix) + postfix + ndirect + 1;
return result;
}
}
/* Represents the range of values belonging to a prefix code:
[offset, offset + 2^nbits) */
typedef struct {
uint16_t offset;
uint8_t nbits;
} BrotliPrefixCodeRange;
/* "Soft-private", it is exported, but not "advertised" as API. */
BROTLI_COMMON_API extern const BROTLI_MODEL("small")
BrotliPrefixCodeRange _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS];
#endif /* BROTLI_COMMON_CONSTANTS_H_ */

View File

@@ -1,112 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup table to map the previous two bytes to a context id.
There are four different context modeling modes defined here:
CONTEXT_LSB6: context id is the least significant 6 bits of the last byte,
CONTEXT_MSB6: context id is the most significant 6 bits of the last byte,
CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text,
CONTEXT_SIGNED: second-order context model tuned for signed integers.
If |p1| and |p2| are the previous two bytes, and |mode| is current context
mode, we calculate the context as:
context = ContextLut(mode)[p1] | ContextLut(mode)[p2 + 256].
For CONTEXT_UTF8 mode, if the previous two bytes are ASCII characters
(i.e. < 128), this will be equivalent to
context = 4 * context1(p1) + context2(p2),
where context1 is based on the previous byte in the following way:
0 : non-ASCII control
1 : \t, \n, \r
2 : space
3 : other punctuation
4 : " '
5 : %
6 : ( < [ {
7 : ) > ] }
8 : , ; :
9 : .
10 : =
11 : number
12 : upper-case vowel
13 : upper-case consonant
14 : lower-case vowel
15 : lower-case consonant
and context2 is based on the second last byte:
0 : control, space
1 : punctuation
2 : upper-case letter, number
3 : lower-case letter
If the last byte is ASCII, and the second last byte is not (in a valid UTF8
stream it will be a continuation byte, value between 128 and 191), the
context is the same as if the second last byte was an ASCII control or space.
If the last byte is a UTF8 lead byte (value >= 192), then the next byte will
be a continuation byte and the context id is 2 or 3 depending on the LSB of
the last byte and to a lesser extent on the second last byte if it is ASCII.
If the last byte is a UTF8 continuation byte, the second last byte can be:
- continuation byte: the next byte is probably ASCII or lead byte (assuming
4-byte UTF8 characters are rare) and the context id is 0 or 1.
- lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1
- lead byte (208 - 255): next byte is continuation byte, context is 2 or 3
The possible value combinations of the previous two bytes, the range of
context ids and the type of the next byte is summarized in the table below:
|--------\-----------------------------------------------------------------|
| \ Last byte |
| Second \---------------------------------------------------------------|
| last byte \ ASCII | cont. byte | lead byte |
| \ (0-127) | (128-191) | (192-) |
|=============|===================|=====================|==================|
| ASCII | next: ASCII/lead | not valid | next: cont. |
| (0-127) | context: 4 - 63 | | context: 2 - 3 |
|-------------|-------------------|---------------------|------------------|
| cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |
| (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |
|-------------|-------------------|---------------------|------------------|
| lead byte | not valid | next: ASCII/lead | not valid |
| (192-207) | | context: 0 - 1 | |
|-------------|-------------------|---------------------|------------------|
| lead byte | not valid | next: cont. | not valid |
| (208-) | | context: 2 - 3 | |
|-------------|-------------------|---------------------|------------------|
*/
#ifndef BROTLI_COMMON_CONTEXT_H_
#define BROTLI_COMMON_CONTEXT_H_
#include "platform.h"
typedef enum ContextType {
CONTEXT_LSB6 = 0,
CONTEXT_MSB6 = 1,
CONTEXT_UTF8 = 2,
CONTEXT_SIGNED = 3
} ContextType;
/* "Soft-private", it is exported, but not "advertised" as API. */
/* Common context lookup table for all context modes. */
BROTLI_COMMON_API extern const uint8_t _kBrotliContextLookupTable[2048];
typedef const uint8_t* ContextLut;
/* typeof(MODE) == ContextType; returns ContextLut */
#define BROTLI_CONTEXT_LUT(MODE) (&_kBrotliContextLookupTable[(MODE) << 9])
/* typeof(LUT) == ContextLut */
#define BROTLI_CONTEXT(P1, P2, LUT) ((LUT)[P1] | ((LUT) + 256)[P2])
#endif /* BROTLI_COMMON_CONTEXT_H_ */

File diff suppressed because one or more lines are too long

Binary file not shown.

View File

@@ -1,64 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "dictionary.h"
#include "platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if !defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
/* Embed kBrotliDictionaryData */
#include "dictionary_inc.h"
static const BROTLI_MODEL("small") BrotliDictionary kBrotliDictionary = {
#else
static BROTLI_MODEL("small") BrotliDictionary kBrotliDictionary = {
#endif
/* size_bits_by_length */
{
0, 0, 0, 0, 10, 10, 11, 11,
10, 10, 10, 10, 10, 9, 9, 8,
7, 7, 8, 7, 7, 6, 6, 5,
5, 0, 0, 0, 0, 0, 0, 0
},
/* offsets_by_length */
{
0, 0, 0, 0, 0, 4096, 9216, 21504,
35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
},
/* data_size == sizeof(kBrotliDictionaryData) */
122784,
/* data */
#if defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
NULL
#else
kBrotliDictionaryData
#endif
};
const BrotliDictionary* BrotliGetDictionary(void) {
return &kBrotliDictionary;
}
void BrotliSetDictionaryData(const uint8_t* data) {
#if defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
if (!!data && !kBrotliDictionary.data) {
kBrotliDictionary.data = data;
}
#else
BROTLI_UNUSED(data); // Appease -Werror=unused-parameter
#endif
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,63 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Collection of static dictionary words. */
#ifndef BROTLI_COMMON_DICTIONARY_H_
#define BROTLI_COMMON_DICTIONARY_H_
#include "platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct BrotliDictionary {
/**
* Number of bits to encode index of dictionary word in a bucket.
*
* Specification: Appendix A. Static Dictionary Data
*
* Words in a dictionary are bucketed by length.
* @c 0 means that there are no words of a given length.
* Dictionary consists of words with length of [4..24] bytes.
* Values at [0..3] and [25..31] indices should not be addressed.
*/
uint8_t size_bits_by_length[32];
/* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
uint32_t offsets_by_length[32];
/* assert(data_size == offsets_by_length[31]) */
size_t data_size;
/* Data array is not bound, and should obey to size_bits_by_length values.
Specified size matches default (RFC 7932) dictionary. Its size is
defined by data_size */
const uint8_t* data;
} BrotliDictionary;
BROTLI_COMMON_API const BrotliDictionary* BrotliGetDictionary(void);
/**
* Sets dictionary data.
*
* When dictionary data is already set / present, this method is no-op.
*
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
* This method is used ONLY in multi-client environment (e.g. C + Java),
* to reduce storage by sharing single dictionary between implementations.
*/
BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data);
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_COMMON_DICTIONARY_H_ */

File diff suppressed because it is too large Load Diff

View File

@@ -1,19 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "platform.h"
/* Default brotli_alloc_func */
void* BrotliDefaultAllocFunc(void* opaque, size_t size) {
BROTLI_UNUSED(opaque);
return malloc(size);
}
/* Default brotli_free_func */
void BrotliDefaultFreeFunc(void* opaque, void* address) {
BROTLI_UNUSED(opaque);
free(address);
}

View File

@@ -1,685 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Macros for compiler / platform specific features and build options.
Build options are:
* BROTLI_BUILD_32_BIT disables 64-bit optimizations
* BROTLI_BUILD_64_BIT forces to use 64-bit optimizations
* BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
* BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
* BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
* BROTLI_BUILD_NO_UNALIGNED_READ_FAST forces off the fast-unaligned-read
optimizations (mainly for testing purposes)
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
* BROTLI_ENABLE_DUMP overrides default "dump" behaviour
*/
#ifndef BROTLI_COMMON_PLATFORM_H_
#define BROTLI_COMMON_PLATFORM_H_
#include <string.h> /* IWYU pragma: export memcmp, memcpy, memset */
#include <stdlib.h> /* IWYU pragma: export exit, free, malloc */
#include <sys/types.h> /* should include endian.h for us */
#include <brotli/port.h> /* IWYU pragma: export */
#include <brotli/types.h> /* IWYU pragma: export */
#if BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
#include <intrin.h>
#endif
#if defined(BROTLI_ENABLE_LOG) || defined(BROTLI_DEBUG)
#include <assert.h>
#include <stdio.h>
#endif
/* The following macros were borrowed from https://github.com/nemequ/hedley
* with permission of original author - Evan Nemerson <evan@nemerson.com> */
/* >>> >>> >>> hedley macros */
/* Define "BROTLI_PREDICT_TRUE" and "BROTLI_PREDICT_FALSE" macros for capable
compilers.
To apply compiler hint, enclose the branching condition into macros, like this:
if (BROTLI_PREDICT_TRUE(zero == 0)) {
// main execution path
} else {
// compiler should place this code outside of main execution path
}
OR:
if (BROTLI_PREDICT_FALSE(something_rare_or_unexpected_happens)) {
// compiler should place this code outside of main execution path
}
*/
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_expect, 3, 0, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
BROTLI_SUNPRO_VERSION_CHECK(5, 15, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
BROTLI_TI_VERSION_CHECK(7, 3, 0) || \
BROTLI_TINYC_VERSION_CHECK(0, 9, 27)
#define BROTLI_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#define BROTLI_PREDICT_FALSE(x) (__builtin_expect(x, 0))
#else
#define BROTLI_PREDICT_FALSE(x) (x)
#define BROTLI_PREDICT_TRUE(x) (x)
#endif
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
!defined(__cplusplus)
#define BROTLI_RESTRICT restrict
#elif BROTLI_GNUC_VERSION_CHECK(3, 1, 0) || \
BROTLI_MSVC_VERSION_CHECK(14, 0, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
BROTLI_PGI_VERSION_CHECK(17, 10, 0) || \
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
BROTLI_IAR_VERSION_CHECK(8, 0, 0) || \
(BROTLI_SUNPRO_VERSION_CHECK(5, 14, 0) && defined(__cplusplus))
#define BROTLI_RESTRICT __restrict
#elif BROTLI_SUNPRO_VERSION_CHECK(5, 3, 0) && !defined(__cplusplus)
#define BROTLI_RESTRICT _Restrict
#else
#define BROTLI_RESTRICT
#endif
#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \
(defined(__cplusplus) && (__cplusplus >= 199711L))
#define BROTLI_MAYBE_INLINE inline
#elif defined(__GNUC_STDC_INLINE__) || defined(__GNUC_GNU_INLINE__) || \
BROTLI_ARM_VERSION_CHECK(6, 2, 0)
#define BROTLI_MAYBE_INLINE __inline__
#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || BROTLI_TI_VERSION_CHECK(8, 0, 0)
#define BROTLI_MAYBE_INLINE __inline
#else
#define BROTLI_MAYBE_INLINE
#endif
#if BROTLI_GNUC_HAS_ATTRIBUTE(always_inline, 4, 0, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
(BROTLI_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__))
#define BROTLI_INLINE BROTLI_MAYBE_INLINE __attribute__((__always_inline__))
#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0)
#define BROTLI_INLINE BROTLI_MAYBE_INLINE __forceinline
#elif BROTLI_TI_VERSION_CHECK(7, 0, 0) && defined(__cplusplus)
#define BROTLI_INLINE BROTLI_MAYBE_INLINE _Pragma("FUNC_ALWAYS_INLINE;")
#elif BROTLI_IAR_VERSION_CHECK(8, 0, 0)
#define BROTLI_INLINE BROTLI_MAYBE_INLINE _Pragma("inline=forced")
#else
#define BROTLI_INLINE BROTLI_MAYBE_INLINE
#endif
#if BROTLI_GNUC_HAS_ATTRIBUTE(noinline, 4, 0, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
(BROTLI_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__))
#define BROTLI_NOINLINE __attribute__((__noinline__))
#elif BROTLI_MSVC_VERSION_CHECK(13, 10, 0)
#define BROTLI_NOINLINE __declspec(noinline)
#elif BROTLI_PGI_VERSION_CHECK(10, 2, 0)
#define BROTLI_NOINLINE _Pragma("noinline")
#elif BROTLI_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus)
#define BROTLI_NOINLINE _Pragma("FUNC_CANNOT_INLINE;")
#elif BROTLI_IAR_VERSION_CHECK(8, 0, 0)
#define BROTLI_NOINLINE _Pragma("inline=never")
#else
#define BROTLI_NOINLINE
#endif
/* <<< <<< <<< end of hedley macros. */
#if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE __attribute__ ((unused))
#else
#define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE
#endif
#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
#define BROTLI_ALIGNED(N) __attribute__((aligned(N)))
#else
#define BROTLI_ALIGNED(N)
#endif
#if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \
(defined(M_ARM) && (M_ARM == 7))
#define BROTLI_TARGET_ARMV7
#endif /* ARMv7 */
#if (defined(__ARM_ARCH) && (__ARM_ARCH == 8)) || \
defined(__aarch64__) || defined(__ARM64_ARCH_8__)
#define BROTLI_TARGET_ARMV8_ANY
#if defined(__ARM_32BIT_STATE)
#define BROTLI_TARGET_ARMV8_32
#elif defined(__ARM_64BIT_STATE)
#define BROTLI_TARGET_ARMV8_64
#endif
#endif /* ARMv8 */
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
#define BROTLI_TARGET_NEON
#endif
#if defined(__i386) || defined(_M_IX86)
#define BROTLI_TARGET_X86
#endif
#if defined(__x86_64__) || defined(_M_X64)
#define BROTLI_TARGET_X64
#endif
#if defined(__PPC64__)
#define BROTLI_TARGET_POWERPC64
#endif
#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 64
#define BROTLI_TARGET_RISCV64
#endif
#if defined(__loongarch_lp64)
#define BROTLI_TARGET_LOONGARCH64
#endif
/* This does not seem to be an indicator of z/Architecture (64-bit); neither
that allows to use unaligned loads. */
#if defined(__s390x__)
#define BROTLI_TARGET_S390X
#endif
#if defined(__mips64)
#define BROTLI_TARGET_MIPS64
#endif
#if defined(__ia64__) || defined(_M_IA64)
#define BROTLI_TARGET_IA64
#endif
#if defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \
defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64) || \
defined(BROTLI_TARGET_LOONGARCH64) || defined(BROTLI_TARGET_MIPS64)
#define BROTLI_TARGET_64_BITS 1
#else
#define BROTLI_TARGET_64_BITS 0
#endif
#if defined(BROTLI_BUILD_64_BIT)
#define BROTLI_64_BITS 1
#elif defined(BROTLI_BUILD_32_BIT)
#define BROTLI_64_BITS 0
#else
#define BROTLI_64_BITS BROTLI_TARGET_64_BITS
#endif
#if (BROTLI_64_BITS)
#define brotli_reg_t uint64_t
#else
#define brotli_reg_t uint32_t
#endif
#if defined(BROTLI_BUILD_BIG_ENDIAN)
#define BROTLI_BIG_ENDIAN 1
#elif defined(BROTLI_BUILD_LITTLE_ENDIAN)
#define BROTLI_LITTLE_ENDIAN 1
#elif defined(BROTLI_BUILD_ENDIAN_NEUTRAL)
/* Just break elif chain. */
#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define BROTLI_LITTLE_ENDIAN 1
#elif defined(_WIN32) || defined(BROTLI_TARGET_X64)
/* Win32 & x64 can currently always be assumed to be little endian */
#define BROTLI_LITTLE_ENDIAN 1
#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#define BROTLI_BIG_ENDIAN 1
/* Likely target platform is iOS / OSX. */
#elif defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)
#define BROTLI_LITTLE_ENDIAN 1
#elif defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)
#define BROTLI_BIG_ENDIAN 1
#endif
#if !defined(BROTLI_LITTLE_ENDIAN)
#define BROTLI_LITTLE_ENDIAN 0
#endif
#if !defined(BROTLI_BIG_ENDIAN)
#define BROTLI_BIG_ENDIAN 0
#endif
#if defined(BROTLI_BUILD_NO_UNALIGNED_READ_FAST)
#define BROTLI_UNALIGNED_READ_FAST (!!0)
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY) || \
defined(BROTLI_TARGET_RISCV64) || defined(BROTLI_TARGET_LOONGARCH64)
/* These targets are known to generate efficient code for unaligned reads
* (e.g. a single instruction, not multiple 1-byte loads, shifted and or'd
* together). */
#define BROTLI_UNALIGNED_READ_FAST (!!1)
#else
#define BROTLI_UNALIGNED_READ_FAST (!!0)
#endif
/* Portable unaligned memory access: read / write values via memcpy. */
#if !defined(BROTLI_USE_PACKED_FOR_UNALIGNED)
#if defined(__mips__) && (!defined(__mips_isa_rev) || __mips_isa_rev < 6)
#define BROTLI_USE_PACKED_FOR_UNALIGNED 1
#else
#define BROTLI_USE_PACKED_FOR_UNALIGNED 0
#endif
#endif /* defined(BROTLI_USE_PACKED_FOR_UNALIGNED) */
#if BROTLI_USE_PACKED_FOR_UNALIGNED
typedef union BrotliPackedValue {
uint16_t u16;
uint32_t u32;
uint64_t u64;
size_t szt;
} __attribute__ ((packed)) BrotliPackedValue;
static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
const BrotliPackedValue* address = (const BrotliPackedValue*)p;
return address->u16;
}
static BROTLI_INLINE uint32_t BrotliUnalignedRead32(const void* p) {
const BrotliPackedValue* address = (const BrotliPackedValue*)p;
return address->u32;
}
static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
const BrotliPackedValue* address = (const BrotliPackedValue*)p;
return address->u64;
}
static BROTLI_INLINE size_t BrotliUnalignedReadSizeT(const void* p) {
const BrotliPackedValue* address = (const BrotliPackedValue*)p;
return address->szt;
}
static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
BrotliPackedValue* address = (BrotliPackedValue*)p;
address->u64 = v;
}
#else /* not BROTLI_USE_PACKED_FOR_UNALIGNED */
static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
uint16_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE uint32_t BrotliUnalignedRead32(const void* p) {
uint32_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE size_t BrotliUnalignedReadSizeT(const void* p) {
size_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
#endif /* BROTLI_USE_PACKED_FOR_UNALIGNED */
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_bswap16, 4, 3, 0)
#define BROTLI_BSWAP16(V) ((uint16_t)__builtin_bswap16(V))
#else
#define BROTLI_BSWAP16(V) ((uint16_t)( \
(((V) & 0xFFU) << 8) | \
(((V) >> 8) & 0xFFU)))
#endif
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_bswap32, 4, 3, 0)
#define BROTLI_BSWAP32(V) ((uint32_t)__builtin_bswap32(V))
#else
#define BROTLI_BSWAP32(V) ((uint32_t)( \
(((V) & 0xFFU) << 24) | (((V) & 0xFF00U) << 8) | \
(((V) >> 8) & 0xFF00U) | (((V) >> 24) & 0xFFU)))
#endif
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_bswap64, 4, 3, 0)
#define BROTLI_BSWAP64(V) ((uint64_t)__builtin_bswap64(V))
#else
#define BROTLI_BSWAP64(V) ((uint64_t)( \
(((V) & 0xFFU) << 56) | (((V) & 0xFF00U) << 40) | \
(((V) & 0xFF0000U) << 24) | (((V) & 0xFF000000U) << 8) | \
(((V) >> 8) & 0xFF000000U) | (((V) >> 24) & 0xFF0000U) | \
(((V) >> 40) & 0xFF00U) | (((V) >> 56) & 0xFFU)))
#endif
#if BROTLI_LITTLE_ENDIAN
/* Straight endianness. Just read / write values. */
#define BROTLI_UNALIGNED_LOAD16LE BrotliUnalignedRead16
#define BROTLI_UNALIGNED_LOAD32LE BrotliUnalignedRead32
#define BROTLI_UNALIGNED_LOAD64LE BrotliUnalignedRead64
#define BROTLI_UNALIGNED_STORE64LE BrotliUnalignedWrite64
#elif BROTLI_BIG_ENDIAN /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE uint16_t BROTLI_UNALIGNED_LOAD16LE(const void* p) {
uint16_t value = BrotliUnalignedRead16(p);
return BROTLI_BSWAP16(value);
}
static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32LE(const void* p) {
uint32_t value = BrotliUnalignedRead32(p);
return BROTLI_BSWAP32(value);
}
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void* p) {
uint64_t value = BrotliUnalignedRead64(p);
return BROTLI_BSWAP64(value);
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
uint64_t value = BROTLI_BSWAP64(v);
BrotliUnalignedWrite64(p, value);
}
#else /* BROTLI_LITTLE_ENDIAN */
/* Read / store values byte-wise; hopefully compiler will understand. */
static BROTLI_INLINE uint16_t BROTLI_UNALIGNED_LOAD16LE(const void* p) {
const uint8_t* in = (const uint8_t*)p;
return (uint16_t)(in[0] | (in[1] << 8));
}
static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32LE(const void* p) {
const uint8_t* in = (const uint8_t*)p;
uint32_t value = (uint32_t)(in[0]);
value |= (uint32_t)(in[1]) << 8;
value |= (uint32_t)(in[2]) << 16;
value |= (uint32_t)(in[3]) << 24;
return value;
}
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void* p) {
const uint8_t* in = (const uint8_t*)p;
uint64_t value = (uint64_t)(in[0]);
value |= (uint64_t)(in[1]) << 8;
value |= (uint64_t)(in[2]) << 16;
value |= (uint64_t)(in[3]) << 24;
value |= (uint64_t)(in[4]) << 32;
value |= (uint64_t)(in[5]) << 40;
value |= (uint64_t)(in[6]) << 48;
value |= (uint64_t)(in[7]) << 56;
return value;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
uint8_t* out = (uint8_t*)p;
out[0] = (uint8_t)v;
out[1] = (uint8_t)(v >> 8);
out[2] = (uint8_t)(v >> 16);
out[3] = (uint8_t)(v >> 24);
out[4] = (uint8_t)(v >> 32);
out[5] = (uint8_t)(v >> 40);
out[6] = (uint8_t)(v >> 48);
out[7] = (uint8_t)(v >> 56);
}
#endif /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE void* BROTLI_UNALIGNED_LOAD_PTR(const void* p) {
void* v;
memcpy(&v, p, sizeof(void*));
return v;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE_PTR(void* p, const void* v) {
memcpy(p, &v, sizeof(void*));
}
/* BROTLI_IS_CONSTANT macros returns true for compile-time constants. */
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_constant_p, 3, 0, 1) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_IS_CONSTANT(x) (!!__builtin_constant_p(x))
#else
#define BROTLI_IS_CONSTANT(x) (!!0)
#endif
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
#define BROTLI_HAS_UBFX (!!1)
#else
#define BROTLI_HAS_UBFX (!!0)
#endif
#if defined(BROTLI_ENABLE_LOG)
#define BROTLI_LOG(x) printf x
#else
#define BROTLI_LOG(x)
#endif
#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
#define BROTLI_ENABLE_DUMP_DEFAULT 1
#define BROTLI_DCHECK(x) assert(x)
#else
#define BROTLI_ENABLE_DUMP_DEFAULT 0
#define BROTLI_DCHECK(x)
#endif
#if !defined(BROTLI_ENABLE_DUMP)
#define BROTLI_ENABLE_DUMP BROTLI_ENABLE_DUMP_DEFAULT
#endif
#if BROTLI_ENABLE_DUMP
static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
fflush(stderr);
}
#define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__)
#else
#define BROTLI_DUMP() (void)(0)
#endif
/* BrotliRBit assumes brotli_reg_t fits native CPU register type. */
#if (BROTLI_64_BITS == BROTLI_TARGET_64_BITS)
/* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */
#if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \
!defined(BROTLI_BUILD_NO_RBIT)
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
/* TODO(eustas): detect ARMv6T2 and enable this code for it. */
static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
brotli_reg_t output;
__asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
return output;
}
#define BROTLI_RBIT(x) BrotliRBit(x)
#endif /* armv7 / armv8 */
#endif /* gcc || clang */
#endif /* brotli_reg_t is native */
#if !defined(BROTLI_RBIT)
static BROTLI_INLINE void BrotliRBit(void) { /* Should break build if used. */ }
#endif /* BROTLI_RBIT */
#define BROTLI_REPEAT_4(X) {X; X; X; X;}
#define BROTLI_REPEAT_5(X) {X; X; X; X; X;}
#define BROTLI_REPEAT_6(X) {X; X; X; X; X; X;}
#define BROTLI_UNUSED(X) (void)(X)
#define BROTLI_MIN_MAX(T) \
static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
BROTLI_MIN_MAX(double) BROTLI_MIN_MAX(float) BROTLI_MIN_MAX(int)
BROTLI_MIN_MAX(size_t) BROTLI_MIN_MAX(uint32_t) BROTLI_MIN_MAX(uint8_t)
#undef BROTLI_MIN_MAX
#define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
#define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
#define BROTLI_SWAP(T, A, I, J) { \
T __brotli_swap_tmp = (A)[(I)]; \
(A)[(I)] = (A)[(J)]; \
(A)[(J)] = __brotli_swap_tmp; \
}
#if BROTLI_64_BITS
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_ctzll, 3, 4, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_TZCNT64 __builtin_ctzll
#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
#if defined(BROTLI_TARGET_X64) && !defined(_M_ARM64EC)
#define BROTLI_TZCNT64 _tzcnt_u64
#else /* BROTLI_TARGET_X64 */
static BROTLI_INLINE uint32_t BrotliBsf64Msvc(uint64_t x) {
uint32_t lsb;
_BitScanForward64(&lsb, x);
return lsb;
}
#define BROTLI_TZCNT64 BrotliBsf64Msvc
#endif /* BROTLI_TARGET_X64 */
#endif /* __builtin_ctzll */
#endif /* BROTLI_64_BITS */
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
#define BROTLI_BSR32(x) (31u ^ (uint32_t)__builtin_clz(x))
#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
static BROTLI_INLINE uint32_t BrotliBsr32Msvc(uint32_t x) {
unsigned long msb;
_BitScanReverse(&msb, x);
return (uint32_t)msb;
}
#define BROTLI_BSR32 BrotliBsr32Msvc
#endif /* __builtin_clz */
/* Default brotli_alloc_func */
BROTLI_COMMON_API void* BrotliDefaultAllocFunc(void* opaque, size_t size);
/* Default brotli_free_func */
BROTLI_COMMON_API void BrotliDefaultFreeFunc(void* opaque, void* address);
/* Circular logical rotates. */
static BROTLI_INLINE uint16_t BrotliRotateRight16(uint16_t const value,
size_t count) {
count &= 0x0F; /* for fickle pattern recognition */
return (value >> count) | (uint16_t)(value << ((0U - count) & 0x0F));
}
static BROTLI_INLINE uint32_t BrotliRotateRight32(uint32_t const value,
size_t count) {
count &= 0x1F; /* for fickle pattern recognition */
return (value >> count) | (uint32_t)(value << ((0U - count) & 0x1F));
}
static BROTLI_INLINE uint64_t BrotliRotateRight64(uint64_t const value,
size_t count) {
count &= 0x3F; /* for fickle pattern recognition */
return (value >> count) | (uint64_t)(value << ((0U - count) & 0x3F));
}
BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) {
BROTLI_UNUSED(&BrotliSuppressUnusedFunctions);
BROTLI_UNUSED(&BrotliUnalignedRead16);
BROTLI_UNUSED(&BrotliUnalignedRead32);
BROTLI_UNUSED(&BrotliUnalignedRead64);
BROTLI_UNUSED(&BrotliUnalignedReadSizeT);
BROTLI_UNUSED(&BrotliUnalignedWrite64);
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD16LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD32LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD64LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE64LE);
BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD_PTR);
BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE_PTR);
BROTLI_UNUSED(&BrotliRBit);
BROTLI_UNUSED(&brotli_min_double);
BROTLI_UNUSED(&brotli_max_double);
BROTLI_UNUSED(&brotli_min_float);
BROTLI_UNUSED(&brotli_max_float);
BROTLI_UNUSED(&brotli_min_int);
BROTLI_UNUSED(&brotli_max_int);
BROTLI_UNUSED(&brotli_min_size_t);
BROTLI_UNUSED(&brotli_max_size_t);
BROTLI_UNUSED(&brotli_min_uint32_t);
BROTLI_UNUSED(&brotli_max_uint32_t);
BROTLI_UNUSED(&brotli_min_uint8_t);
BROTLI_UNUSED(&brotli_max_uint8_t);
BROTLI_UNUSED(&BrotliDefaultAllocFunc);
BROTLI_UNUSED(&BrotliDefaultFreeFunc);
BROTLI_UNUSED(&BrotliRotateRight16);
BROTLI_UNUSED(&BrotliRotateRight32);
BROTLI_UNUSED(&BrotliRotateRight64);
#if BROTLI_ENABLE_DUMP
BROTLI_UNUSED(&BrotliDump);
#endif
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && \
!defined(_M_ARM64EC)
/* _mm_prefetch() is not defined outside of x86/x64 */
/* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
#include <mmintrin.h>
#define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
#define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
#elif BROTLI_GNUC_HAS_BUILTIN(__builtin_prefetch, 3, 1, 0)
#define PREFETCH_L1(ptr) \
__builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
#define PREFETCH_L2(ptr) \
__builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
#elif defined(__aarch64__)
#define PREFETCH_L1(ptr) \
do { \
__asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); \
} while (0)
#define PREFETCH_L2(ptr) \
do { \
__asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); \
} while (0)
#else
#define PREFETCH_L1(ptr) \
do { \
(void)(ptr); \
} while (0) /* disabled */
#define PREFETCH_L2(ptr) \
do { \
(void)(ptr); \
} while (0) /* disabled */
#endif
/* The SIMD matchers are only faster at certain quality levels. */
#if defined(_M_X64) && defined(BROTLI_TZCNT64)
#define BROTLI_MAX_SIMD_QUALITY 7
#elif defined(BROTLI_TZCNT64)
#define BROTLI_MAX_SIMD_QUALITY 6
#endif
}
#if defined(_MSC_VER)
#define BROTLI_CRASH() __debugbreak(), (void)abort()
#elif BROTLI_GNUC_HAS_BUILTIN(__builtin_trap, 3, 0, 0)
#define BROTLI_CRASH() (void)__builtin_trap()
#else
#define BROTLI_CRASH() (void)abort()
#endif
/* Make BROTLI_TEST=0 act same as undefined. */
#if defined(BROTLI_TEST) && ((1-BROTLI_TEST-1) == 0)
#undef BROTLI_TEST
#endif
#if !defined(BROTLI_MODEL) && BROTLI_GNUC_HAS_ATTRIBUTE(model, 3, 0, 3) && \
!defined(BROTLI_TARGET_IA64) && !defined(BROTLI_TARGET_LOONGARCH64)
#define BROTLI_MODEL(M) __attribute__((model(M)))
#else
#define BROTLI_MODEL(M) /* M */
#endif
#if !defined(BROTLI_COLD) && BROTLI_GNUC_HAS_ATTRIBUTE(cold, 4, 3, 0)
#define BROTLI_COLD __attribute__((cold))
#else
#define BROTLI_COLD /* cold */
#endif
#endif /* BROTLI_COMMON_PLATFORM_H_ */

View File

@@ -1,517 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Shared Dictionary definition and utilities. */
#include <brotli/shared_dictionary.h>
#include "dictionary.h"
#include "platform.h"
#include "shared_dictionary_internal.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if defined(BROTLI_EXPERIMENTAL)
#define BROTLI_NUM_ENCODED_LENGTHS (SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH \
- SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH + 1)
/* Max allowed by spec */
#define BROTLI_MAX_SIZE_BITS 15u
/* Returns BROTLI_TRUE on success, BROTLI_FALSE on failure. */
static BROTLI_BOOL ReadBool(const uint8_t* encoded, size_t size, size_t* pos,
BROTLI_BOOL* result) {
uint8_t value;
size_t position = *pos;
if (position >= size) return BROTLI_FALSE; /* past file end */
value = encoded[position++];
if (value > 1) return BROTLI_FALSE; /* invalid bool */
*result = TO_BROTLI_BOOL(value);
*pos = position;
return BROTLI_TRUE; /* success */
}
/* Returns BROTLI_TRUE on success, BROTLI_FALSE on failure. */
static BROTLI_BOOL ReadUint8(const uint8_t* encoded, size_t size, size_t* pos,
uint8_t* result) {
size_t position = *pos;
if (position + sizeof(uint8_t) > size) return BROTLI_FALSE;
*result = encoded[position++];
*pos = position;
return BROTLI_TRUE;
}
/* Returns BROTLI_TRUE on success, BROTLI_FALSE on failure. */
static BROTLI_BOOL ReadUint16(const uint8_t* encoded, size_t size, size_t* pos,
uint16_t* result) {
size_t position = *pos;
if (position + sizeof(uint16_t) > size) return BROTLI_FALSE;
*result = BROTLI_UNALIGNED_LOAD16LE(&encoded[position]);
position += 2;
*pos = position;
return BROTLI_TRUE;
}
/* Reads a varint into a uint32_t, and returns error if it's too large */
/* Returns BROTLI_TRUE on success, BROTLI_FALSE on failure. */
static BROTLI_BOOL ReadVarint32(const uint8_t* encoded, size_t size,
size_t* pos, uint32_t* result) {
int num = 0;
uint8_t byte;
*result = 0;
for (;;) {
if (*pos >= size) return BROTLI_FALSE;
byte = encoded[(*pos)++];
if (num == 4 && byte > 15) return BROTLI_FALSE;
*result |= (uint32_t)(byte & 127) << (num * 7);
if (byte < 128) return BROTLI_TRUE;
num++;
}
}
/* Returns the total length of word list. */
static size_t BrotliSizeBitsToOffsets(const uint8_t* size_bits_by_length,
uint32_t* offsets_by_length) {
uint32_t pos = 0;
uint32_t i;
for (i = 0; i <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; i++) {
offsets_by_length[i] = pos;
if (size_bits_by_length[i] != 0) {
pos += i << size_bits_by_length[i];
}
}
return pos;
}
static BROTLI_BOOL ParseWordList(size_t size, const uint8_t* encoded,
size_t* pos, BrotliDictionary* out) {
size_t offset;
size_t i;
size_t position = *pos;
if (position + BROTLI_NUM_ENCODED_LENGTHS > size) {
return BROTLI_FALSE;
}
memset(out->size_bits_by_length, 0, SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH);
memcpy(out->size_bits_by_length + SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH,
&encoded[position], BROTLI_NUM_ENCODED_LENGTHS);
for (i = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
i <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; i++) {
if (out->size_bits_by_length[i] > BROTLI_MAX_SIZE_BITS) {
return BROTLI_FALSE;
}
}
position += BROTLI_NUM_ENCODED_LENGTHS;
offset = BrotliSizeBitsToOffsets(
out->size_bits_by_length, out->offsets_by_length);
out->data = &encoded[position];
out->data_size = offset;
position += offset;
if (position > size) return BROTLI_FALSE;
*pos = position;
return BROTLI_TRUE;
}
/* Computes the cutOffTransforms of a BrotliTransforms which already has the
transforms data correctly filled in. */
static void ComputeCutoffTransforms(BrotliTransforms* transforms) {
uint32_t i;
for (i = 0; i < BROTLI_TRANSFORMS_MAX_CUT_OFF + 1; i++) {
transforms->cutOffTransforms[i] = -1;
}
for (i = 0; i < transforms->num_transforms; i++) {
const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, i);
uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, i);
const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, i);
if (type <= BROTLI_TRANSFORM_OMIT_LAST_9 && *prefix == 0 && *suffix == 0 &&
transforms->cutOffTransforms[type] == -1) {
transforms->cutOffTransforms[type] = (int16_t)i;
}
}
}
static BROTLI_BOOL ParsePrefixSuffixTable(size_t size, const uint8_t* encoded,
size_t* pos, BrotliTransforms* out, uint16_t* out_table,
size_t* out_table_size) {
size_t position = *pos;
size_t offset = 0;
size_t stringlet_count = 0; /* NUM_PREFIX_SUFFIX */
size_t data_length = 0;
/* PREFIX_SUFFIX_LENGTH */
if (!ReadUint16(encoded, size, &position, &out->prefix_suffix_size)) {
return BROTLI_FALSE;
}
data_length = out->prefix_suffix_size;
/* Must at least have space for null terminator. */
if (data_length < 1) return BROTLI_FALSE;
out->prefix_suffix = &encoded[position];
if (position + data_length >= size) return BROTLI_FALSE;
while (BROTLI_TRUE) {
/* STRING_LENGTH */
size_t stringlet_len = encoded[position + offset];
out_table[stringlet_count] = (uint16_t)offset;
stringlet_count++;
offset++;
if (stringlet_len == 0) {
if (offset == data_length) {
break;
} else {
return BROTLI_FALSE;
}
}
if (stringlet_count > 255) return BROTLI_FALSE;
offset += stringlet_len;
if (offset >= data_length) return BROTLI_FALSE;
}
position += data_length;
*pos = position;
*out_table_size = (uint16_t)stringlet_count;
return BROTLI_TRUE;
}
static BROTLI_BOOL ParseTransformsList(size_t size, const uint8_t* encoded,
size_t* pos, BrotliTransforms* out, uint16_t* prefix_suffix_table,
size_t* prefix_suffix_count) {
uint32_t i;
BROTLI_BOOL has_params = BROTLI_FALSE;
BROTLI_BOOL prefix_suffix_ok = BROTLI_FALSE;
size_t position = *pos;
size_t stringlet_cnt = 0;
if (position >= size) return BROTLI_FALSE;
prefix_suffix_ok = ParsePrefixSuffixTable(
size, encoded, &position, out, prefix_suffix_table, &stringlet_cnt);
if (!prefix_suffix_ok) return BROTLI_FALSE;
out->prefix_suffix_map = prefix_suffix_table;
*prefix_suffix_count = stringlet_cnt;
out->num_transforms = encoded[position++];
out->transforms = &encoded[position];
position += (size_t)out->num_transforms * 3;
if (position > size) return BROTLI_FALSE;
/* Check for errors and read extra parameters. */
for (i = 0; i < out->num_transforms; i++) {
uint8_t prefix_id = BROTLI_TRANSFORM_PREFIX_ID(out, i);
uint8_t type = BROTLI_TRANSFORM_TYPE(out, i);
uint8_t suffix_id = BROTLI_TRANSFORM_SUFFIX_ID(out, i);
if (prefix_id >= stringlet_cnt) return BROTLI_FALSE;
if (type >= BROTLI_NUM_TRANSFORM_TYPES) return BROTLI_FALSE;
if (suffix_id >= stringlet_cnt) return BROTLI_FALSE;
if (type == BROTLI_TRANSFORM_SHIFT_FIRST ||
type == BROTLI_TRANSFORM_SHIFT_ALL) {
has_params = BROTLI_TRUE;
}
}
if (has_params) {
out->params = &encoded[position];
position += (size_t)out->num_transforms * 2;
if (position > size) return BROTLI_FALSE;
for (i = 0; i < out->num_transforms; i++) {
uint8_t type = BROTLI_TRANSFORM_TYPE(out, i);
if (type != BROTLI_TRANSFORM_SHIFT_FIRST &&
type != BROTLI_TRANSFORM_SHIFT_ALL) {
if (out->params[i * 2] != 0 || out->params[i * 2 + 1] != 0) {
return BROTLI_FALSE;
}
}
}
} else {
out->params = NULL;
}
ComputeCutoffTransforms(out);
*pos = position;
return BROTLI_TRUE;
}
static BROTLI_BOOL DryParseDictionary(const uint8_t* encoded,
size_t size, uint32_t* num_prefix, BROTLI_BOOL* is_custom_static_dict) {
size_t pos = 0;
uint32_t chunk_size = 0;
uint8_t num_word_lists;
uint8_t num_transform_lists;
*is_custom_static_dict = BROTLI_FALSE;
*num_prefix = 0;
/* Skip magic header bytes. */
pos += 2;
/* LZ77_DICTIONARY_LENGTH */
if (!ReadVarint32(encoded, size, &pos, &chunk_size)) return BROTLI_FALSE;
if (chunk_size != 0) {
/* This limitation is not specified but the 32-bit Brotli decoder for now */
if (chunk_size > 1073741823) return BROTLI_FALSE;
*num_prefix = 1;
if (pos + chunk_size > size) return BROTLI_FALSE;
pos += chunk_size;
}
if (!ReadUint8(encoded, size, &pos, &num_word_lists)) {
return BROTLI_FALSE;
}
if (!ReadUint8(encoded, size, &pos, &num_transform_lists)) {
return BROTLI_FALSE;
}
if (num_word_lists > 0 || num_transform_lists > 0) {
*is_custom_static_dict = BROTLI_TRUE;
}
return BROTLI_TRUE;
}
static BROTLI_BOOL ParseDictionary(const uint8_t* encoded, size_t size,
BrotliSharedDictionary* dict) {
uint32_t i;
size_t pos = 0;
uint32_t chunk_size = 0;
size_t total_prefix_suffix_count = 0;
size_t transform_list_start[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
uint16_t temporary_prefix_suffix_table[256];
/* Skip magic header bytes. */
pos += 2;
/* LZ77_DICTIONARY_LENGTH */
if (!ReadVarint32(encoded, size, &pos, &chunk_size)) return BROTLI_FALSE;
if (chunk_size != 0) {
if (pos + chunk_size > size) return BROTLI_FALSE;
dict->prefix_size[dict->num_prefix] = chunk_size;
dict->prefix[dict->num_prefix] = &encoded[pos];
dict->num_prefix++;
/* LZ77_DICTIONARY_LENGTH bytes. */
pos += chunk_size;
}
/* NUM_WORD_LISTS */
if (!ReadUint8(encoded, size, &pos, &dict->num_word_lists)) {
return BROTLI_FALSE;
}
if (dict->num_word_lists > SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS) {
return BROTLI_FALSE;
}
if (dict->num_word_lists != 0) {
dict->words_instances = (BrotliDictionary*)dict->alloc_func(
dict->memory_manager_opaque,
dict->num_word_lists * sizeof(*dict->words_instances));
if (!dict->words_instances) return BROTLI_FALSE; /* OOM */
}
for (i = 0; i < dict->num_word_lists; i++) {
if (!ParseWordList(size, encoded, &pos, &dict->words_instances[i])) {
return BROTLI_FALSE;
}
}
/* NUM_TRANSFORM_LISTS */
if (!ReadUint8(encoded, size, &pos, &dict->num_transform_lists)) {
return BROTLI_FALSE;
}
if (dict->num_transform_lists > SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS) {
return BROTLI_FALSE;
}
if (dict->num_transform_lists != 0) {
dict->transforms_instances = (BrotliTransforms*)dict->alloc_func(
dict->memory_manager_opaque,
dict->num_transform_lists * sizeof(*dict->transforms_instances));
if (!dict->transforms_instances) return BROTLI_FALSE; /* OOM */
}
for (i = 0; i < dict->num_transform_lists; i++) {
BROTLI_BOOL ok = BROTLI_FALSE;
size_t prefix_suffix_count = 0;
transform_list_start[i] = pos;
dict->transforms_instances[i].prefix_suffix_map =
temporary_prefix_suffix_table;
ok = ParseTransformsList(
size, encoded, &pos, &dict->transforms_instances[i],
temporary_prefix_suffix_table, &prefix_suffix_count);
if (!ok) return BROTLI_FALSE;
total_prefix_suffix_count += prefix_suffix_count;
}
if (total_prefix_suffix_count != 0) {
dict->prefix_suffix_maps = (uint16_t*)dict->alloc_func(
dict->memory_manager_opaque,
total_prefix_suffix_count * sizeof(*dict->prefix_suffix_maps));
if (!dict->prefix_suffix_maps) return BROTLI_FALSE; /* OOM */
}
total_prefix_suffix_count = 0;
for (i = 0; i < dict->num_transform_lists; i++) {
size_t prefix_suffix_count = 0;
size_t position = transform_list_start[i];
uint16_t* prefix_suffix_map =
&dict->prefix_suffix_maps[total_prefix_suffix_count];
BROTLI_BOOL ok = ParsePrefixSuffixTable(
size, encoded, &position, &dict->transforms_instances[i],
prefix_suffix_map, &prefix_suffix_count);
if (!ok) return BROTLI_FALSE;
dict->transforms_instances[i].prefix_suffix_map = prefix_suffix_map;
total_prefix_suffix_count += prefix_suffix_count;
}
if (dict->num_word_lists != 0 || dict->num_transform_lists != 0) {
if (!ReadUint8(encoded, size, &pos, &dict->num_dictionaries)) {
return BROTLI_FALSE;
}
if (dict->num_dictionaries == 0 ||
dict->num_dictionaries > SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS) {
return BROTLI_FALSE;
}
for (i = 0; i < dict->num_dictionaries; i++) {
uint8_t words_index;
uint8_t transforms_index;
if (!ReadUint8(encoded, size, &pos, &words_index)) {
return BROTLI_FALSE;
}
if (words_index > dict->num_word_lists) return BROTLI_FALSE;
if (!ReadUint8(encoded, size, &pos, &transforms_index)) {
return BROTLI_FALSE;
}
if (transforms_index > dict->num_transform_lists) return BROTLI_FALSE;
dict->words[i] = words_index == dict->num_word_lists ?
BrotliGetDictionary() : &dict->words_instances[words_index];
dict->transforms[i] = transforms_index == dict->num_transform_lists ?
BrotliGetTransforms(): &dict->transforms_instances[transforms_index];
}
/* CONTEXT_ENABLED */
if (!ReadBool(encoded, size, &pos, &dict->context_based)) {
return BROTLI_FALSE;
}
/* CONTEXT_MAP */
if (dict->context_based) {
for (i = 0; i < SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS; i++) {
if (!ReadUint8(encoded, size, &pos, &dict->context_map[i])) {
return BROTLI_FALSE;
}
if (dict->context_map[i] >= dict->num_dictionaries) {
return BROTLI_FALSE;
}
}
}
} else {
dict->context_based = BROTLI_FALSE;
dict->num_dictionaries = 1;
dict->words[0] = BrotliGetDictionary();
dict->transforms[0] = BrotliGetTransforms();
}
return BROTLI_TRUE;
}
/* Decodes shared dictionary and verifies correctness.
Returns BROTLI_TRUE if dictionary is valid, BROTLI_FALSE otherwise.
The BrotliSharedDictionary must already have been initialized. If the
BrotliSharedDictionary already contains data, compound dictionaries
will be appended, but an error will be returned if it already has
custom words or transforms.
TODO(lode): link to RFC for shared brotli once published. */
static BROTLI_BOOL DecodeSharedDictionary(
const uint8_t* encoded, size_t size, BrotliSharedDictionary* dict) {
uint32_t num_prefix = 0;
BROTLI_BOOL is_custom_static_dict = BROTLI_FALSE;
BROTLI_BOOL has_custom_static_dict =
dict->num_word_lists > 0 || dict->num_transform_lists > 0;
/* Check magic header bytes. */
if (size < 2) return BROTLI_FALSE;
if (encoded[0] != 0x91 || encoded[1] != 0) return BROTLI_FALSE;
if (!DryParseDictionary(encoded, size, &num_prefix, &is_custom_static_dict)) {
return BROTLI_FALSE;
}
if (num_prefix + dict->num_prefix > SHARED_BROTLI_MAX_COMPOUND_DICTS) {
return BROTLI_FALSE;
}
/* Cannot combine different static dictionaries, only prefix dictionaries */
if (has_custom_static_dict && is_custom_static_dict) return BROTLI_FALSE;
return ParseDictionary(encoded, size, dict);
}
#endif /* BROTLI_EXPERIMENTAL */
void BrotliSharedDictionaryDestroyInstance(
BrotliSharedDictionary* dict) {
if (!dict) {
return;
} else {
brotli_free_func free_func = dict->free_func;
void* opaque = dict->memory_manager_opaque;
/* Cleanup. */
free_func(opaque, dict->words_instances);
free_func(opaque, dict->transforms_instances);
free_func(opaque, dict->prefix_suffix_maps);
/* Self-destruction. */
free_func(opaque, dict);
}
}
BROTLI_BOOL BrotliSharedDictionaryAttach(
BrotliSharedDictionary* dict, BrotliSharedDictionaryType type,
size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)]) {
if (!dict) {
return BROTLI_FALSE;
}
#if defined(BROTLI_EXPERIMENTAL)
if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) {
return DecodeSharedDictionary(data, data_size, dict);
}
#endif /* BROTLI_EXPERIMENTAL */
if (type == BROTLI_SHARED_DICTIONARY_RAW) {
if (dict->num_prefix >= SHARED_BROTLI_MAX_COMPOUND_DICTS) {
return BROTLI_FALSE;
}
dict->prefix_size[dict->num_prefix] = data_size;
dict->prefix[dict->num_prefix] = data;
dict->num_prefix++;
return BROTLI_TRUE;
}
return BROTLI_FALSE;
}
BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
BrotliSharedDictionary* dict = 0;
if (!alloc_func && !free_func) {
dict = (BrotliSharedDictionary*)malloc(sizeof(BrotliSharedDictionary));
} else if (alloc_func && free_func) {
dict = (BrotliSharedDictionary*)alloc_func(
opaque, sizeof(BrotliSharedDictionary));
}
if (dict == 0) {
return 0;
}
/* TODO(eustas): explicitly initialize all the fields? */
memset(dict, 0, sizeof(BrotliSharedDictionary));
dict->context_based = BROTLI_FALSE;
dict->num_dictionaries = 1;
dict->num_word_lists = 0;
dict->num_transform_lists = 0;
dict->words[0] = BrotliGetDictionary();
dict->transforms[0] = BrotliGetTransforms();
dict->alloc_func = alloc_func ? alloc_func : BrotliDefaultAllocFunc;
dict->free_func = free_func ? free_func : BrotliDefaultFreeFunc;
dict->memory_manager_opaque = opaque;
return dict;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,75 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* (Transparent) Shared Dictionary definition. */
#ifndef BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_
#define BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_
#include <brotli/shared_dictionary.h>
#include "dictionary.h"
#include "platform.h"
#include "transform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
struct BrotliSharedDictionaryStruct {
/* LZ77 prefixes (compound dictionary). */
uint32_t num_prefix; /* max SHARED_BROTLI_MAX_COMPOUND_DICTS */
size_t prefix_size[SHARED_BROTLI_MAX_COMPOUND_DICTS];
const uint8_t* prefix[SHARED_BROTLI_MAX_COMPOUND_DICTS];
/* If set, the context map is used to select word and transform list from 64
contexts, if not set, the context map is not used and only words[0] and
transforms[0] are to be used. */
BROTLI_BOOL context_based;
uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
/* Amount of word_list+transform_list combinations. */
uint8_t num_dictionaries;
/* Must use num_dictionaries values. */
const BrotliDictionary* words[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
/* Must use num_dictionaries values. */
const BrotliTransforms* transforms[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
/* Amount of custom word lists. May be 0 if only Brotli's built-in is used */
uint8_t num_word_lists;
/* Contents of the custom words lists. Must be NULL if num_word_lists is 0. */
BrotliDictionary* words_instances;
/* Amount of custom transform lists. May be 0 if only Brotli's built-in is
used */
uint8_t num_transform_lists;
/* Contents of the custom transform lists. Must be NULL if num_transform_lists
is 0. */
BrotliTransforms* transforms_instances;
/* Concatenated prefix_suffix_maps of the custom transform lists. Must be NULL
if num_transform_lists is 0. */
uint16_t* prefix_suffix_maps;
/* Memory management */
brotli_alloc_func alloc_func;
brotli_free_func free_func;
void* memory_manager_opaque;
};
typedef struct BrotliSharedDictionaryStruct BrotliSharedDictionaryInternal;
#define BrotliSharedDictionary BrotliSharedDictionaryInternal
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_COMMON_SHARED_DICTIONARY_INTERNAL_H_ */

View File

@@ -1,56 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/*
Central point for static initialization.
In case of "lazy" mode `BrotliXxxLazyStaticInit` is not provided by the
library. Embedder is responsible for providing it. This function should call
`BrotliXxxLazyStaticInitInner` on the first invocation. This function should
not return until execution of `BrotliXxxLazyStaticInitInner` is finished.
In C or before C++11 it is possible to call `BrotliXxxLazyStaticInitInner`
on start-up path and then `BrotliEncoderLazyStaticInit` is could be no-op;
another option is to use available thread execution controls to meet the
requirements. For possible C++11 implementation see static_init_lazy.cc.
*/
#ifndef THIRD_PARTY_BROTLI_COMMON_STATIC_INIT_H_
#define THIRD_PARTY_BROTLI_COMMON_STATIC_INIT_H_
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Static data is "initialized" in compile time. */
#define BROTLI_STATIC_INIT_NONE 0
/* Static data is initialized before "main". */
#define BROTLI_STATIC_INIT_EARLY 1
/* Static data is initialized when first encoder is created. */
#define BROTLI_STATIC_INIT_LAZY 2
#define BROTLI_STATIC_INIT_DEFAULT BROTLI_STATIC_INIT_NONE
#if !defined(BROTLI_STATIC_INIT)
#define BROTLI_STATIC_INIT BROTLI_STATIC_INIT_DEFAULT
#endif
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE) && \
(BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_EARLY) && \
(BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_LAZY)
#error Invalid value for BROTLI_STATIC_INIT
#endif
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_EARLY)
#if defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
#error BROTLI_STATIC_INIT_EARLY will fail with BROTLI_EXTERNAL_DICTIONARY_DATA
#endif
#endif /* BROTLI_STATIC_INIT */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif // THIRD_PARTY_BROTLI_COMMON_STATIC_INIT_H_

View File

@@ -1,293 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "platform.h"
#include "transform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* RFC 7932 transforms string data */
static const BROTLI_MODEL("small") char kPrefixSuffix[217] =
"\1 \2, \10 of the \4 of \2s \1.\5 and \4 "
/* 0x _0 _2 __5 _E _3 _6 _8 _E */
"in \1\"\4 to \2\">\1\n\2. \1]\5 for \3 a \6 "
/* 2x _3_ _5 _A_ _D_ _F _2 _4 _A _E */
"that \1\'\6 with \6 from \4 by \1(\6. T"
/* 4x _5_ _7 _E _5 _A _C */
"he \4 on \4 as \4 is \4ing \2\n\t\1:\3ed "
/* 6x _3 _8 _D _2 _7_ _ _A _C */
"\2=\"\4 at \3ly \1,\2=\'\5.com/\7. This \5"
/* 8x _0 _ _3 _8 _C _E _ _1 _7 _F */
" not \3er \3al \4ful \4ive \5less \4es"
/* Ax _5 _9 _D _2 _7 _D */
"t \4ize \2\xc2\xa0\4ous \5 the \2e "; /* \0 - implicit trailing zero. */
/* Cx _2 _7___ ___ _A _F _5 _8 */
static const BROTLI_MODEL("small") uint16_t kPrefixSuffixMap[50] = {
0x00, 0x02, 0x05, 0x0E, 0x13, 0x16, 0x18, 0x1E, 0x23, 0x25,
0x2A, 0x2D, 0x2F, 0x32, 0x34, 0x3A, 0x3E, 0x45, 0x47, 0x4E,
0x55, 0x5A, 0x5C, 0x63, 0x68, 0x6D, 0x72, 0x77, 0x7A, 0x7C,
0x80, 0x83, 0x88, 0x8C, 0x8E, 0x91, 0x97, 0x9F, 0xA5, 0xA9,
0xAD, 0xB2, 0xB7, 0xBD, 0xC2, 0xC7, 0xCA, 0xCF, 0xD5, 0xD8
};
/* RFC 7932 transforms */
static const BROTLI_MODEL("small") uint8_t kTransformsData[] = {
49, BROTLI_TRANSFORM_IDENTITY, 49,
49, BROTLI_TRANSFORM_IDENTITY, 0,
0, BROTLI_TRANSFORM_IDENTITY, 0,
49, BROTLI_TRANSFORM_OMIT_FIRST_1, 49,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 0,
49, BROTLI_TRANSFORM_IDENTITY, 47,
0, BROTLI_TRANSFORM_IDENTITY, 49,
4, BROTLI_TRANSFORM_IDENTITY, 0,
49, BROTLI_TRANSFORM_IDENTITY, 3,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 49,
49, BROTLI_TRANSFORM_IDENTITY, 6,
49, BROTLI_TRANSFORM_OMIT_FIRST_2, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_1, 49,
1, BROTLI_TRANSFORM_IDENTITY, 0,
49, BROTLI_TRANSFORM_IDENTITY, 1,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 0,
49, BROTLI_TRANSFORM_IDENTITY, 7,
49, BROTLI_TRANSFORM_IDENTITY, 9,
48, BROTLI_TRANSFORM_IDENTITY, 0,
49, BROTLI_TRANSFORM_IDENTITY, 8,
49, BROTLI_TRANSFORM_IDENTITY, 5,
49, BROTLI_TRANSFORM_IDENTITY, 10,
49, BROTLI_TRANSFORM_IDENTITY, 11,
49, BROTLI_TRANSFORM_OMIT_LAST_3, 49,
49, BROTLI_TRANSFORM_IDENTITY, 13,
49, BROTLI_TRANSFORM_IDENTITY, 14,
49, BROTLI_TRANSFORM_OMIT_FIRST_3, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_2, 49,
49, BROTLI_TRANSFORM_IDENTITY, 15,
49, BROTLI_TRANSFORM_IDENTITY, 16,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 49,
49, BROTLI_TRANSFORM_IDENTITY, 12,
5, BROTLI_TRANSFORM_IDENTITY, 49,
0, BROTLI_TRANSFORM_IDENTITY, 1,
49, BROTLI_TRANSFORM_OMIT_FIRST_4, 49,
49, BROTLI_TRANSFORM_IDENTITY, 18,
49, BROTLI_TRANSFORM_IDENTITY, 17,
49, BROTLI_TRANSFORM_IDENTITY, 19,
49, BROTLI_TRANSFORM_IDENTITY, 20,
49, BROTLI_TRANSFORM_OMIT_FIRST_5, 49,
49, BROTLI_TRANSFORM_OMIT_FIRST_6, 49,
47, BROTLI_TRANSFORM_IDENTITY, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_4, 49,
49, BROTLI_TRANSFORM_IDENTITY, 22,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 49,
49, BROTLI_TRANSFORM_IDENTITY, 23,
49, BROTLI_TRANSFORM_IDENTITY, 24,
49, BROTLI_TRANSFORM_IDENTITY, 25,
49, BROTLI_TRANSFORM_OMIT_LAST_7, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_1, 26,
49, BROTLI_TRANSFORM_IDENTITY, 27,
49, BROTLI_TRANSFORM_IDENTITY, 28,
0, BROTLI_TRANSFORM_IDENTITY, 12,
49, BROTLI_TRANSFORM_IDENTITY, 29,
49, BROTLI_TRANSFORM_OMIT_FIRST_9, 49,
49, BROTLI_TRANSFORM_OMIT_FIRST_7, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_6, 49,
49, BROTLI_TRANSFORM_IDENTITY, 21,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 1,
49, BROTLI_TRANSFORM_OMIT_LAST_8, 49,
49, BROTLI_TRANSFORM_IDENTITY, 31,
49, BROTLI_TRANSFORM_IDENTITY, 32,
47, BROTLI_TRANSFORM_IDENTITY, 3,
49, BROTLI_TRANSFORM_OMIT_LAST_5, 49,
49, BROTLI_TRANSFORM_OMIT_LAST_9, 49,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 1,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 8,
5, BROTLI_TRANSFORM_IDENTITY, 21,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 0,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 10,
49, BROTLI_TRANSFORM_IDENTITY, 30,
0, BROTLI_TRANSFORM_IDENTITY, 5,
35, BROTLI_TRANSFORM_IDENTITY, 49,
47, BROTLI_TRANSFORM_IDENTITY, 2,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 17,
49, BROTLI_TRANSFORM_IDENTITY, 36,
49, BROTLI_TRANSFORM_IDENTITY, 33,
5, BROTLI_TRANSFORM_IDENTITY, 0,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 21,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 5,
49, BROTLI_TRANSFORM_IDENTITY, 37,
0, BROTLI_TRANSFORM_IDENTITY, 30,
49, BROTLI_TRANSFORM_IDENTITY, 38,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 0,
49, BROTLI_TRANSFORM_IDENTITY, 39,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 49,
49, BROTLI_TRANSFORM_IDENTITY, 34,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 8,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 12,
0, BROTLI_TRANSFORM_IDENTITY, 21,
49, BROTLI_TRANSFORM_IDENTITY, 40,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 12,
49, BROTLI_TRANSFORM_IDENTITY, 41,
49, BROTLI_TRANSFORM_IDENTITY, 42,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 17,
49, BROTLI_TRANSFORM_IDENTITY, 43,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 5,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 10,
0, BROTLI_TRANSFORM_IDENTITY, 34,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 33,
49, BROTLI_TRANSFORM_IDENTITY, 44,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 5,
45, BROTLI_TRANSFORM_IDENTITY, 49,
0, BROTLI_TRANSFORM_IDENTITY, 33,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 30,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 30,
49, BROTLI_TRANSFORM_IDENTITY, 46,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 1,
49, BROTLI_TRANSFORM_UPPERCASE_FIRST, 34,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 33,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 30,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 1,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 33,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 21,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 12,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 5,
49, BROTLI_TRANSFORM_UPPERCASE_ALL, 34,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 12,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 30,
0, BROTLI_TRANSFORM_UPPERCASE_ALL, 34,
0, BROTLI_TRANSFORM_UPPERCASE_FIRST, 34,
};
static const BROTLI_MODEL("small")
BrotliTransforms kBrotliTransforms = {
sizeof(kPrefixSuffix),
(const uint8_t*)kPrefixSuffix,
kPrefixSuffixMap,
sizeof(kTransformsData) / (3 * sizeof(kTransformsData[0])),
kTransformsData,
NULL, /* no extra parameters */
{0, 12, 27, 23, 42, 63, 56, 48, 59, 64}
};
const BrotliTransforms* BrotliGetTransforms(void) {
return &kBrotliTransforms;
}
static int ToUpperCase(uint8_t* p) {
if (p[0] < 0xC0) {
if (p[0] >= 'a' && p[0] <= 'z') {
p[0] ^= 32;
}
return 1;
}
/* An overly simplified uppercasing model for UTF-8. */
if (p[0] < 0xE0) {
p[1] ^= 32;
return 2;
}
/* An arbitrary transform for three byte characters. */
p[2] ^= 5;
return 3;
}
static int Shift(uint8_t* word, int word_len, uint16_t parameter) {
/* Limited sign extension: scalar < (1 << 24). */
uint32_t scalar =
(parameter & 0x7FFFu) + (0x1000000u - (parameter & 0x8000u));
if (word[0] < 0x80) {
/* 1-byte rune / 0sssssss / 7 bit scalar (ASCII). */
scalar += (uint32_t)word[0];
word[0] = (uint8_t)(scalar & 0x7Fu);
return 1;
} else if (word[0] < 0xC0) {
/* Continuation / 10AAAAAA. */
return 1;
} else if (word[0] < 0xE0) {
/* 2-byte rune / 110sssss AAssssss / 11 bit scalar. */
if (word_len < 2) return 1;
scalar += (uint32_t)((word[1] & 0x3Fu) | ((word[0] & 0x1Fu) << 6u));
word[0] = (uint8_t)(0xC0 | ((scalar >> 6u) & 0x1F));
word[1] = (uint8_t)((word[1] & 0xC0) | (scalar & 0x3F));
return 2;
} else if (word[0] < 0xF0) {
/* 3-byte rune / 1110ssss AAssssss BBssssss / 16 bit scalar. */
if (word_len < 3) return word_len;
scalar += (uint32_t)((word[2] & 0x3Fu) | ((word[1] & 0x3Fu) << 6u) |
((word[0] & 0x0Fu) << 12u));
word[0] = (uint8_t)(0xE0 | ((scalar >> 12u) & 0x0F));
word[1] = (uint8_t)((word[1] & 0xC0) | ((scalar >> 6u) & 0x3F));
word[2] = (uint8_t)((word[2] & 0xC0) | (scalar & 0x3F));
return 3;
} else if (word[0] < 0xF8) {
/* 4-byte rune / 11110sss AAssssss BBssssss CCssssss / 21 bit scalar. */
if (word_len < 4) return word_len;
scalar += (uint32_t)((word[3] & 0x3Fu) | ((word[2] & 0x3Fu) << 6u) |
((word[1] & 0x3Fu) << 12u) | ((word[0] & 0x07u) << 18u));
word[0] = (uint8_t)(0xF0 | ((scalar >> 18u) & 0x07));
word[1] = (uint8_t)((word[1] & 0xC0) | ((scalar >> 12u) & 0x3F));
word[2] = (uint8_t)((word[2] & 0xC0) | ((scalar >> 6u) & 0x3F));
word[3] = (uint8_t)((word[3] & 0xC0) | (scalar & 0x3F));
return 4;
}
return 1;
}
int BrotliTransformDictionaryWord(uint8_t* dst, const uint8_t* word, int len,
const BrotliTransforms* transforms, int transform_idx) {
int idx = 0;
const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, transform_idx);
uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, transform_idx);
const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, transform_idx);
{
int prefix_len = *prefix++;
while (prefix_len--) { dst[idx++] = *prefix++; }
}
{
const int t = type;
int i = 0;
if (t <= BROTLI_TRANSFORM_OMIT_LAST_9) {
len -= t;
} else if (t >= BROTLI_TRANSFORM_OMIT_FIRST_1
&& t <= BROTLI_TRANSFORM_OMIT_FIRST_9) {
int skip = t - (BROTLI_TRANSFORM_OMIT_FIRST_1 - 1);
word += skip;
len -= skip;
}
while (i < len) { dst[idx++] = word[i++]; }
if (t == BROTLI_TRANSFORM_UPPERCASE_FIRST) {
ToUpperCase(&dst[idx - len]);
} else if (t == BROTLI_TRANSFORM_UPPERCASE_ALL) {
uint8_t* uppercase = &dst[idx - len];
while (len > 0) {
int step = ToUpperCase(uppercase);
uppercase += step;
len -= step;
}
} else if (t == BROTLI_TRANSFORM_SHIFT_FIRST) {
uint16_t param = (uint16_t)(transforms->params[transform_idx * 2]
+ (transforms->params[transform_idx * 2 + 1] << 8u));
Shift(&dst[idx - len], len, param);
} else if (t == BROTLI_TRANSFORM_SHIFT_ALL) {
uint16_t param = (uint16_t)(transforms->params[transform_idx * 2]
+ (transforms->params[transform_idx * 2 + 1] << 8u));
uint8_t* shift = &dst[idx - len];
while (len > 0) {
int step = Shift(shift, len, param);
shift += step;
len -= step;
}
}
}
{
int suffix_len = *suffix++;
while (suffix_len--) { dst[idx++] = *suffix++; }
return idx;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,84 +0,0 @@
/* transforms is a part of ABI, but not API.
It means that there are some functions that are supposed to be in "common"
library, but header itself is not placed into include/brotli. This way,
aforementioned functions will be available only to brotli internals.
*/
#ifndef BROTLI_COMMON_TRANSFORM_H_
#define BROTLI_COMMON_TRANSFORM_H_
#include "platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
enum BrotliWordTransformType {
BROTLI_TRANSFORM_IDENTITY = 0,
BROTLI_TRANSFORM_OMIT_LAST_1 = 1,
BROTLI_TRANSFORM_OMIT_LAST_2 = 2,
BROTLI_TRANSFORM_OMIT_LAST_3 = 3,
BROTLI_TRANSFORM_OMIT_LAST_4 = 4,
BROTLI_TRANSFORM_OMIT_LAST_5 = 5,
BROTLI_TRANSFORM_OMIT_LAST_6 = 6,
BROTLI_TRANSFORM_OMIT_LAST_7 = 7,
BROTLI_TRANSFORM_OMIT_LAST_8 = 8,
BROTLI_TRANSFORM_OMIT_LAST_9 = 9,
BROTLI_TRANSFORM_UPPERCASE_FIRST = 10,
BROTLI_TRANSFORM_UPPERCASE_ALL = 11,
BROTLI_TRANSFORM_OMIT_FIRST_1 = 12,
BROTLI_TRANSFORM_OMIT_FIRST_2 = 13,
BROTLI_TRANSFORM_OMIT_FIRST_3 = 14,
BROTLI_TRANSFORM_OMIT_FIRST_4 = 15,
BROTLI_TRANSFORM_OMIT_FIRST_5 = 16,
BROTLI_TRANSFORM_OMIT_FIRST_6 = 17,
BROTLI_TRANSFORM_OMIT_FIRST_7 = 18,
BROTLI_TRANSFORM_OMIT_FIRST_8 = 19,
BROTLI_TRANSFORM_OMIT_FIRST_9 = 20,
BROTLI_TRANSFORM_SHIFT_FIRST = 21,
BROTLI_TRANSFORM_SHIFT_ALL = 22,
BROTLI_NUM_TRANSFORM_TYPES /* Counts transforms, not a transform itself. */
};
#define BROTLI_TRANSFORMS_MAX_CUT_OFF BROTLI_TRANSFORM_OMIT_LAST_9
typedef struct BrotliTransforms {
uint16_t prefix_suffix_size;
/* Last character must be null, so prefix_suffix_size must be at least 1. */
const uint8_t* prefix_suffix;
const uint16_t* prefix_suffix_map;
uint32_t num_transforms;
/* Each entry is a [prefix_id, transform, suffix_id] triplet. */
const uint8_t* transforms;
/* Shift for BROTLI_TRANSFORM_SHIFT_FIRST and BROTLI_TRANSFORM_SHIFT_ALL,
must be NULL if and only if no such transforms are present. */
const uint8_t* params;
/* Indices of transforms like ["", BROTLI_TRANSFORM_OMIT_LAST_#, ""].
0-th element corresponds to ["", BROTLI_TRANSFORM_IDENTITY, ""].
-1, if cut-off transform does not exist. */
int16_t cutOffTransforms[BROTLI_TRANSFORMS_MAX_CUT_OFF + 1];
} BrotliTransforms;
/* T is BrotliTransforms*; result is uint8_t. */
#define BROTLI_TRANSFORM_PREFIX_ID(T, I) ((T)->transforms[((I) * 3) + 0])
#define BROTLI_TRANSFORM_TYPE(T, I) ((T)->transforms[((I) * 3) + 1])
#define BROTLI_TRANSFORM_SUFFIX_ID(T, I) ((T)->transforms[((I) * 3) + 2])
/* T is BrotliTransforms*; result is const uint8_t*. */
#define BROTLI_TRANSFORM_PREFIX(T, I) (&(T)->prefix_suffix[ \
(T)->prefix_suffix_map[BROTLI_TRANSFORM_PREFIX_ID(T, I)]])
#define BROTLI_TRANSFORM_SUFFIX(T, I) (&(T)->prefix_suffix[ \
(T)->prefix_suffix_map[BROTLI_TRANSFORM_SUFFIX_ID(T, I)]])
BROTLI_COMMON_API const BrotliTransforms* BrotliGetTransforms(void);
BROTLI_COMMON_API int BrotliTransformDictionaryWord(
uint8_t* dst, const uint8_t* word, int len,
const BrotliTransforms* transforms, int transform_idx);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_COMMON_TRANSFORM_H_ */

View File

@@ -1,51 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Version definition. */
#ifndef BROTLI_COMMON_VERSION_H_
#define BROTLI_COMMON_VERSION_H_
/* Compose 3 components into a single number. In a hexadecimal representation
B and C components occupy exactly 3 digits. */
#define BROTLI_MAKE_HEX_VERSION(A, B, C) ((A << 24) | (B << 12) | C)
/* Those macros should only be used when library is compiled together with
the client. If library is dynamically linked, use BrotliDecoderVersion and
BrotliEncoderVersion methods. */
#define BROTLI_VERSION_MAJOR 1
#define BROTLI_VERSION_MINOR 2
#define BROTLI_VERSION_PATCH 0
#define BROTLI_VERSION BROTLI_MAKE_HEX_VERSION( \
BROTLI_VERSION_MAJOR, BROTLI_VERSION_MINOR, BROTLI_VERSION_PATCH)
/* This macro is used by build system to produce Libtool-friendly soname. See
https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
Version evolution rules:
- interfaces added (or change is compatible) -> current+1:0:age+1
- interfaces removed (or changed is incompatible) -> current+1:0:0
- interfaces not changed -> current:revision+1:age
*/
#define BROTLI_ABI_CURRENT 3
#define BROTLI_ABI_REVISION 0
#define BROTLI_ABI_AGE 2
#if BROTLI_VERSION_MAJOR != (BROTLI_ABI_CURRENT - BROTLI_ABI_AGE)
#error ABI/API version inconsistency
#endif
#if BROTLI_VERSION_MINOR != BROTLI_ABI_AGE
#error ABI/API version inconsistency
#endif
#if BROTLI_VERSION_PATCH != BROTLI_ABI_REVISION
#error ABI/API version inconsistency
#endif
#endif /* BROTLI_COMMON_VERSION_H_ */

View File

@@ -1,77 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Bit reading helpers */
#include "bit_reader.h"
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
const BROTLI_MODEL("small")
brotli_reg_t kBrotliBitMask[33] = { 0x00000000,
0x00000001, 0x00000003, 0x00000007, 0x0000000F,
0x0000001F, 0x0000003F, 0x0000007F, 0x000000FF,
0x000001FF, 0x000003FF, 0x000007FF, 0x00000FFF,
0x00001FFF, 0x00003FFF, 0x00007FFF, 0x0000FFFF,
0x0001FFFF, 0x0003FFFF, 0x0007FFFF, 0x000FFFFF,
0x001FFFFF, 0x003FFFFF, 0x007FFFFF, 0x00FFFFFF,
0x01FFFFFF, 0x03FFFFFF, 0x07FFFFFF, 0x0FFFFFFF,
0x1FFFFFFF, 0x3FFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF
};
void BrotliInitBitReader(BrotliBitReader* const br) {
br->val_ = 0;
br->bit_pos_ = 0;
}
BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
enough space in accumulator to fix alignment. */
if (BROTLI_UNALIGNED_READ_FAST) {
aligned_read_mask = 0;
}
if (BrotliGetAvailableBits(br) == 0) {
br->val_ = 0;
if (!BrotliPullByte(br)) {
return BROTLI_FALSE;
}
}
while ((((size_t)br->next_in) & aligned_read_mask) != 0) {
if (!BrotliPullByte(br)) {
/* If we consumed all the input, we don't care about the alignment. */
return BROTLI_TRUE;
}
}
return BROTLI_TRUE;
}
BROTLI_BOOL BrotliSafeReadBits32Slow(BrotliBitReader* const br,
brotli_reg_t n_bits, brotli_reg_t* val) {
brotli_reg_t low_val;
brotli_reg_t high_val;
BrotliBitReaderState memento;
BROTLI_DCHECK(n_bits <= 32);
BROTLI_DCHECK(n_bits > 24);
BrotliBitReaderSaveState(br, &memento);
if (!BrotliSafeReadBits(br, 16, &low_val) ||
!BrotliSafeReadBits(br, n_bits - 16, &high_val)) {
BrotliBitReaderRestoreState(br, &memento);
return BROTLI_FALSE;
}
*val = low_val | (high_val << 16);
return BROTLI_TRUE;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,419 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Bit reading helpers */
#ifndef BROTLI_DEC_BIT_READER_H_
#define BROTLI_DEC_BIT_READER_H_
#include "../common/constants.h"
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define BROTLI_SHORT_FILL_BIT_WINDOW_READ (sizeof(brotli_reg_t) >> 1)
/* 162 bits + 7 bytes */
#define BROTLI_FAST_INPUT_SLACK 28
BROTLI_INTERNAL extern const brotli_reg_t kBrotliBitMask[33];
static BROTLI_INLINE brotli_reg_t BitMask(brotli_reg_t n) {
if (BROTLI_IS_CONSTANT(n) || BROTLI_HAS_UBFX) {
/* Masking with this expression turns to a single
"Unsigned Bit Field Extract" UBFX instruction on ARM. */
return ~(~((brotli_reg_t)0) << n);
} else {
return kBrotliBitMask[n];
}
}
typedef struct {
brotli_reg_t val_; /* pre-fetched bits */
brotli_reg_t bit_pos_; /* current bit-reading position in val_ */
const uint8_t* next_in; /* the byte we're reading from */
const uint8_t* guard_in; /* position from which "fast-path" is prohibited */
const uint8_t* last_in; /* == next_in + avail_in */
} BrotliBitReader;
typedef struct {
brotli_reg_t val_;
brotli_reg_t bit_pos_;
const uint8_t* next_in;
size_t avail_in;
} BrotliBitReaderState;
/* Initializes the BrotliBitReader fields. */
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* br);
/* Ensures that accumulator is not empty.
May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
Returns BROTLI_FALSE if data is required but there is no input available.
For !BROTLI_UNALIGNED_READ_FAST this function also prepares bit reader for
aligned reading. */
BROTLI_INTERNAL BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* br);
/* Fallback for BrotliSafeReadBits32. Extracted as noninlined method to unburden
the main code-path. Never called for RFC brotli streams, required only for
"large-window" mode and other extensions. */
BROTLI_INTERNAL BROTLI_NOINLINE BROTLI_BOOL BrotliSafeReadBits32Slow(
BrotliBitReader* br, brotli_reg_t n_bits, brotli_reg_t* val);
static BROTLI_INLINE size_t
BrotliBitReaderGetAvailIn(BrotliBitReader* const br) {
return (size_t)(br->last_in - br->next_in);
}
static BROTLI_INLINE void BrotliBitReaderSaveState(
BrotliBitReader* const from, BrotliBitReaderState* to) {
to->val_ = from->val_;
to->bit_pos_ = from->bit_pos_;
to->next_in = from->next_in;
to->avail_in = BrotliBitReaderGetAvailIn(from);
}
static BROTLI_INLINE void BrotliBitReaderSetInput(
BrotliBitReader* const br, const uint8_t* next_in, size_t avail_in) {
br->next_in = next_in;
br->last_in = (avail_in == 0) ? next_in : (next_in + avail_in);
if (avail_in + 1 > BROTLI_FAST_INPUT_SLACK) {
br->guard_in = next_in + (avail_in + 1 - BROTLI_FAST_INPUT_SLACK);
} else {
br->guard_in = next_in;
}
}
static BROTLI_INLINE void BrotliBitReaderRestoreState(
BrotliBitReader* const to, BrotliBitReaderState* from) {
to->val_ = from->val_;
to->bit_pos_ = from->bit_pos_;
to->next_in = from->next_in;
BrotliBitReaderSetInput(to, from->next_in, from->avail_in);
}
static BROTLI_INLINE brotli_reg_t BrotliGetAvailableBits(
const BrotliBitReader* br) {
return br->bit_pos_;
}
/* Returns amount of unread bytes the bit reader still has buffered from the
BrotliInput, including whole bytes in br->val_. Result is capped with
maximal ring-buffer size (larger number won't be utilized anyway). */
static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
static const size_t kCap = (size_t)1 << BROTLI_LARGE_MAX_WBITS;
size_t avail_in = BrotliBitReaderGetAvailIn(br);
if (avail_in > kCap) return kCap;
return avail_in + (BrotliGetAvailableBits(br) >> 3);
}
/* Checks if there is at least |num| bytes left in the input ring-buffer
(excluding the bits remaining in br->val_). */
static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount(
BrotliBitReader* const br) {
return TO_BROTLI_BOOL(br->next_in < br->guard_in);
}
/* Load more bits into accumulator. */
static BROTLI_INLINE brotli_reg_t BrotliBitReaderLoadBits(brotli_reg_t val,
brotli_reg_t new_bits,
brotli_reg_t count,
brotli_reg_t offset) {
BROTLI_DCHECK(
!((val >> offset) & ~new_bits & ~(~((brotli_reg_t)0) << count)));
(void)count;
return val | (new_bits << offset);
}
/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
Precondition: accumulator contains at least 1 bit.
|n_bits| should be in the range [1..24] for regular build. For portable
non-64-bit little-endian build only 16 bits are safe to request. */
static BROTLI_INLINE void BrotliFillBitWindow(
BrotliBitReader* const br, brotli_reg_t n_bits) {
#if (BROTLI_64_BITS)
if (BROTLI_UNALIGNED_READ_FAST && BROTLI_IS_CONSTANT(n_bits) &&
(n_bits <= 8)) {
brotli_reg_t bit_pos = br->bit_pos_;
if (bit_pos <= 8) {
br->val_ = BrotliBitReaderLoadBits(br->val_,
BROTLI_UNALIGNED_LOAD64LE(br->next_in), 56, bit_pos);
br->bit_pos_ = bit_pos + 56;
br->next_in += 7;
}
} else if (BROTLI_UNALIGNED_READ_FAST && BROTLI_IS_CONSTANT(n_bits) &&
(n_bits <= 16)) {
brotli_reg_t bit_pos = br->bit_pos_;
if (bit_pos <= 16) {
br->val_ = BrotliBitReaderLoadBits(br->val_,
BROTLI_UNALIGNED_LOAD64LE(br->next_in), 48, bit_pos);
br->bit_pos_ = bit_pos + 48;
br->next_in += 6;
}
} else {
brotli_reg_t bit_pos = br->bit_pos_;
if (bit_pos <= 32) {
br->val_ = BrotliBitReaderLoadBits(br->val_,
(uint64_t)BROTLI_UNALIGNED_LOAD32LE(br->next_in), 32, bit_pos);
br->bit_pos_ = bit_pos + 32;
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
}
}
#else
if (BROTLI_UNALIGNED_READ_FAST && BROTLI_IS_CONSTANT(n_bits) &&
(n_bits <= 8)) {
brotli_reg_t bit_pos = br->bit_pos_;
if (bit_pos <= 8) {
br->val_ = BrotliBitReaderLoadBits(br->val_,
BROTLI_UNALIGNED_LOAD32LE(br->next_in), 24, bit_pos);
br->bit_pos_ = bit_pos + 24;
br->next_in += 3;
}
} else {
brotli_reg_t bit_pos = br->bit_pos_;
if (bit_pos <= 16) {
br->val_ = BrotliBitReaderLoadBits(br->val_,
(uint32_t)BROTLI_UNALIGNED_LOAD16LE(br->next_in), 16, bit_pos);
br->bit_pos_ = bit_pos + 16;
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
}
}
#endif
}
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) {
BrotliFillBitWindow(br, 17);
}
/* Tries to pull one byte of input to accumulator.
Returns BROTLI_FALSE if there is no input available. */
static BROTLI_INLINE BROTLI_BOOL BrotliPullByte(BrotliBitReader* const br) {
if (br->next_in == br->last_in) {
return BROTLI_FALSE;
}
br->val_ = BrotliBitReaderLoadBits(br->val_,
(brotli_reg_t)*br->next_in, 8, br->bit_pos_);
br->bit_pos_ += 8;
++br->next_in;
return BROTLI_TRUE;
}
/* Returns currently available bits.
The number of valid bits could be calculated by BrotliGetAvailableBits. */
static BROTLI_INLINE brotli_reg_t BrotliGetBitsUnmasked(
BrotliBitReader* const br) {
return br->val_;
}
/* Like BrotliGetBits, but does not mask the result.
The result contains at least 16 valid bits. */
static BROTLI_INLINE brotli_reg_t BrotliGet16BitsUnmasked(
BrotliBitReader* const br) {
BrotliFillBitWindow(br, 16);
return (brotli_reg_t)BrotliGetBitsUnmasked(br);
}
/* Returns the specified number of bits from |br| without advancing bit
position. */
static BROTLI_INLINE brotli_reg_t BrotliGetBits(
BrotliBitReader* const br, brotli_reg_t n_bits) {
BrotliFillBitWindow(br, n_bits);
return BrotliGetBitsUnmasked(br) & BitMask(n_bits);
}
/* Tries to peek the specified amount of bits. Returns BROTLI_FALSE, if there
is not enough input. */
static BROTLI_INLINE BROTLI_BOOL BrotliSafeGetBits(
BrotliBitReader* const br, brotli_reg_t n_bits, brotli_reg_t* val) {
while (BrotliGetAvailableBits(br) < n_bits) {
if (!BrotliPullByte(br)) {
return BROTLI_FALSE;
}
}
*val = BrotliGetBitsUnmasked(br) & BitMask(n_bits);
return BROTLI_TRUE;
}
/* Advances the bit pos by |n_bits|. */
static BROTLI_INLINE void BrotliDropBits(
BrotliBitReader* const br, brotli_reg_t n_bits) {
br->bit_pos_ -= n_bits;
br->val_ >>= n_bits;
}
/* Make sure that there are no spectre bits in accumulator.
This is important for the cases when some bytes are skipped
(i.e. never placed into accumulator). */
static BROTLI_INLINE void BrotliBitReaderNormalize(BrotliBitReader* br) {
/* Actually, it is enough to normalize when br->bit_pos_ == 0 */
if (br->bit_pos_ < (sizeof(brotli_reg_t) << 3u)) {
br->val_ &= (((brotli_reg_t)1) << br->bit_pos_) - 1;
}
}
static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
brotli_reg_t unused_bytes = BrotliGetAvailableBits(br) >> 3;
brotli_reg_t unused_bits = unused_bytes << 3;
br->next_in =
(unused_bytes == 0) ? br->next_in : (br->next_in - unused_bytes);
br->bit_pos_ -= unused_bits;
BrotliBitReaderNormalize(br);
}
/* Reads the specified number of bits from |br| and advances the bit pos.
Precondition: accumulator MUST contain at least |n_bits|. */
static BROTLI_INLINE void BrotliTakeBits(BrotliBitReader* const br,
brotli_reg_t n_bits,
brotli_reg_t* val) {
*val = BrotliGetBitsUnmasked(br) & BitMask(n_bits);
BROTLI_LOG(("[BrotliTakeBits] %d %d %d val: %6x\n",
(int)BrotliBitReaderGetAvailIn(br), (int)br->bit_pos_,
(int)n_bits, (int)*val));
BrotliDropBits(br, n_bits);
}
/* Reads the specified number of bits from |br| and advances the bit pos.
Assumes that there is enough input to perform BrotliFillBitWindow.
Up to 24 bits are allowed to be requested from this method. */
static BROTLI_INLINE brotli_reg_t BrotliReadBits24(
BrotliBitReader* const br, brotli_reg_t n_bits) {
BROTLI_DCHECK(n_bits <= 24);
if (BROTLI_64_BITS || (n_bits <= 16)) {
brotli_reg_t val;
BrotliFillBitWindow(br, n_bits);
BrotliTakeBits(br, n_bits, &val);
return val;
} else {
brotli_reg_t low_val;
brotli_reg_t high_val;
BrotliFillBitWindow(br, 16);
BrotliTakeBits(br, 16, &low_val);
BrotliFillBitWindow(br, 8);
BrotliTakeBits(br, n_bits - 16, &high_val);
return low_val | (high_val << 16);
}
}
/* Same as BrotliReadBits24, but allows reading up to 32 bits. */
static BROTLI_INLINE brotli_reg_t BrotliReadBits32(
BrotliBitReader* const br, brotli_reg_t n_bits) {
BROTLI_DCHECK(n_bits <= 32);
if (BROTLI_64_BITS || (n_bits <= 16)) {
brotli_reg_t val;
BrotliFillBitWindow(br, n_bits);
BrotliTakeBits(br, n_bits, &val);
return val;
} else {
brotli_reg_t low_val;
brotli_reg_t high_val;
BrotliFillBitWindow(br, 16);
BrotliTakeBits(br, 16, &low_val);
BrotliFillBitWindow(br, 16);
BrotliTakeBits(br, n_bits - 16, &high_val);
return low_val | (high_val << 16);
}
}
/* Tries to read the specified amount of bits. Returns BROTLI_FALSE, if there
is not enough input. |n_bits| MUST be positive.
Up to 24 bits are allowed to be requested from this method. */
static BROTLI_INLINE BROTLI_BOOL BrotliSafeReadBits(
BrotliBitReader* const br, brotli_reg_t n_bits, brotli_reg_t* val) {
BROTLI_DCHECK(n_bits <= 24);
while (BrotliGetAvailableBits(br) < n_bits) {
if (!BrotliPullByte(br)) {
return BROTLI_FALSE;
}
}
BrotliTakeBits(br, n_bits, val);
return BROTLI_TRUE;
}
/* Same as BrotliSafeReadBits, but allows reading up to 32 bits. */
static BROTLI_INLINE BROTLI_BOOL BrotliSafeReadBits32(
BrotliBitReader* const br, brotli_reg_t n_bits, brotli_reg_t* val) {
BROTLI_DCHECK(n_bits <= 32);
if (BROTLI_64_BITS || (n_bits <= 24)) {
while (BrotliGetAvailableBits(br) < n_bits) {
if (!BrotliPullByte(br)) {
return BROTLI_FALSE;
}
}
BrotliTakeBits(br, n_bits, val);
return BROTLI_TRUE;
} else {
return BrotliSafeReadBits32Slow(br, n_bits, val);
}
}
/* Advances the bit reader position to the next byte boundary and verifies
that any skipped bits are set to zero. */
static BROTLI_INLINE BROTLI_BOOL BrotliJumpToByteBoundary(BrotliBitReader* br) {
brotli_reg_t pad_bits_count = BrotliGetAvailableBits(br) & 0x7;
brotli_reg_t pad_bits = 0;
if (pad_bits_count != 0) {
BrotliTakeBits(br, pad_bits_count, &pad_bits);
}
BrotliBitReaderNormalize(br);
return TO_BROTLI_BOOL(pad_bits == 0);
}
static BROTLI_INLINE void BrotliDropBytes(BrotliBitReader* br, size_t num) {
/* Check detour is legal: accumulator must to be empty. */
BROTLI_DCHECK(br->bit_pos_ == 0);
BROTLI_DCHECK(br->val_ == 0);
br->next_in += num;
}
/* Copies remaining input bytes stored in the bit reader to the output. Value
|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
warmed up again after this. */
static BROTLI_INLINE void BrotliCopyBytes(uint8_t* dest,
BrotliBitReader* br, size_t num) {
while (BrotliGetAvailableBits(br) >= 8 && num > 0) {
*dest = (uint8_t)BrotliGetBitsUnmasked(br);
BrotliDropBits(br, 8);
++dest;
--num;
}
BrotliBitReaderNormalize(br);
if (num > 0) {
memcpy(dest, br->next_in, num);
BrotliDropBytes(br, num);
}
}
BROTLI_UNUSED_FUNCTION void BrotliBitReaderSuppressUnusedFunctions(void) {
BROTLI_UNUSED(&BrotliBitReaderSuppressUnusedFunctions);
BROTLI_UNUSED(&BrotliBitReaderGetAvailIn);
BROTLI_UNUSED(&BrotliBitReaderLoadBits);
BROTLI_UNUSED(&BrotliBitReaderRestoreState);
BROTLI_UNUSED(&BrotliBitReaderSaveState);
BROTLI_UNUSED(&BrotliBitReaderSetInput);
BROTLI_UNUSED(&BrotliBitReaderUnload);
BROTLI_UNUSED(&BrotliCheckInputAmount);
BROTLI_UNUSED(&BrotliCopyBytes);
BROTLI_UNUSED(&BrotliFillBitWindow16);
BROTLI_UNUSED(&BrotliGet16BitsUnmasked);
BROTLI_UNUSED(&BrotliGetBits);
BROTLI_UNUSED(&BrotliGetRemainingBytes);
BROTLI_UNUSED(&BrotliJumpToByteBoundary);
BROTLI_UNUSED(&BrotliReadBits24);
BROTLI_UNUSED(&BrotliReadBits32);
BROTLI_UNUSED(&BrotliSafeGetBits);
BROTLI_UNUSED(&BrotliSafeReadBits);
BROTLI_UNUSED(&BrotliSafeReadBits32);
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_BIT_READER_H_ */

File diff suppressed because it is too large Load Diff

View File

@@ -1,120 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Utilities for building Huffman decoding tables. */
#ifndef BROTLI_DEC_HUFFMAN_H_
#define BROTLI_DEC_HUFFMAN_H_
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH 15
/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */
#define BROTLI_HUFFMAN_MAX_SIZE_26 396
/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */
#define BROTLI_HUFFMAN_MAX_SIZE_258 632
/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */
#define BROTLI_HUFFMAN_MAX_SIZE_272 646
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5
#if ((defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_32)) && \
BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0))
#define BROTLI_HUFFMAN_CODE_FAST_LOAD
#endif
#if !defined(BROTLI_HUFFMAN_CODE_FAST_LOAD)
/* Do not create this struct directly - use the ConstructHuffmanCode
* constructor below! */
typedef struct {
uint8_t bits; /* number of bits used for this symbol */
uint16_t value; /* symbol value or table offset */
} HuffmanCode;
static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits,
const uint16_t value) {
HuffmanCode h;
h.bits = bits;
h.value = value;
return h;
}
/* Please use the following macros to optimize HuffmanCode accesses in hot
* paths.
*
* For example, assuming |table| contains a HuffmanCode pointer:
*
* BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
* BROTLI_HC_ADJUST_TABLE_INDEX(table, index_into_table);
* *bits = BROTLI_HC_GET_BITS(table);
* *value = BROTLI_HC_GET_VALUE(table);
* BROTLI_HC_ADJUST_TABLE_INDEX(table, offset);
* *bits2 = BROTLI_HC_GET_BITS(table);
* *value2 = BROTLI_HC_GET_VALUE(table);
*
*/
#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H)
#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V)
/* These must be given a HuffmanCode pointer! */
#define BROTLI_HC_FAST_LOAD_BITS(H) (H->bits)
#define BROTLI_HC_FAST_LOAD_VALUE(H) (H->value)
#else /* BROTLI_HUFFMAN_CODE_FAST_LOAD */
typedef BROTLI_ALIGNED(4) uint32_t HuffmanCode;
static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits,
const uint16_t value) {
return (HuffmanCode) ((value & 0xFFFF) << 16) | (bits & 0xFF);
}
#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) uint32_t __fastload_##H = (*H)
#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V); __fastload_##H = (*H)
/* These must be given a HuffmanCode pointer! */
#define BROTLI_HC_FAST_LOAD_BITS(H) ((__fastload_##H) & 0xFF)
#define BROTLI_HC_FAST_LOAD_VALUE(H) ((__fastload_##H) >> 16)
#endif /* BROTLI_HUFFMAN_CODE_FAST_LOAD */
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
const uint8_t* const code_lengths, uint16_t* count);
/* Builds Huffman lookup table assuming code lengths are in symbol order.
Returns size of resulting table. */
BROTLI_INTERNAL uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
int root_bits, const uint16_t* const symbol_lists, uint16_t* count);
/* Builds a simple Huffman table. The |num_symbols| parameter is to be
interpreted as follows: 0 means 1 symbol, 1 means 2 symbols,
2 means 3 symbols, 3 means 4 symbols with lengths [2, 2, 2, 2],
4 means 4 symbols with lengths [1, 2, 3, 3]. */
BROTLI_INTERNAL uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
int root_bits, uint16_t* symbols, uint32_t num_symbols);
/* Contains a collection of Huffman trees with the same alphabet size. */
/* alphabet_size_limit is needed due to simple codes, since
log2(alphabet_size_max) could be greater than log2(alphabet_size_limit). */
typedef struct {
HuffmanCode** htrees;
HuffmanCode* codes;
uint16_t alphabet_size_max;
uint16_t alphabet_size_limit;
uint16_t num_htrees;
} HuffmanTreeGroup;
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_HUFFMAN_H_ */

View File

@@ -1,67 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "prefix.h"
#include "../common/platform.h" /* IWYU pragma: keep */
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/constants.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_NONE)
/* Embed kCmdLut. */
#include "prefix_inc.h"
#else
BROTLI_COLD BROTLI_BOOL BrotliDecoderInitCmdLut(CmdLutElement* items) {
static const uint8_t kInsertLengthExtraBits[24] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03,
0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x18};
static const uint8_t kCopyLengthExtraBits[24] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02,
0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x18};
static const uint8_t kCellPos[11] = {0, 1, 0, 1, 8, 9, 2, 16, 10, 17, 18};
uint16_t insert_length_offsets[24];
uint16_t copy_length_offsets[24];
insert_length_offsets[0] = 0;
copy_length_offsets[0] = 2;
for (size_t i = 0; i < 23; ++i) {
insert_length_offsets[i + 1] =
insert_length_offsets[i] + (uint16_t)(1u << kInsertLengthExtraBits[i]);
copy_length_offsets[i + 1] =
copy_length_offsets[i] + (uint16_t)(1u << kCopyLengthExtraBits[i]);
}
for (size_t symbol = 0; symbol < BROTLI_NUM_COMMAND_SYMBOLS; ++symbol) {
CmdLutElement* item = items + symbol;
const size_t cell_idx = symbol >> 6;
const size_t cell_pos = kCellPos[cell_idx];
const size_t copy_code = ((cell_pos << 3) & 0x18) + (symbol & 0x7);
const uint16_t copy_len_offset = copy_length_offsets[copy_code];
const size_t insert_code = (cell_pos & 0x18) + ((symbol >> 3) & 0x7);
item->copy_len_extra_bits = kCopyLengthExtraBits[copy_code];
item->context = (copy_len_offset > 4) ? 3 : ((uint8_t)copy_len_offset - 2);
item->copy_len_offset = copy_len_offset;
item->distance_code = (cell_idx >= 2) ? -1 : 0;
item->insert_len_extra_bits = kInsertLengthExtraBits[insert_code];
item->insert_len_offset = insert_length_offsets[insert_code];
}
return BROTLI_TRUE;
}
BROTLI_MODEL("small")
CmdLutElement kCmdLut[BROTLI_NUM_COMMAND_SYMBOLS];
#endif /* BROTLI_STATIC_INIT */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,43 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup tables to map prefix codes to value ranges. This is used during
decoding of the block lengths, literal insertion lengths and copy lengths. */
#ifndef BROTLI_DEC_PREFIX_H_
#define BROTLI_DEC_PREFIX_H_
#include "../common/constants.h"
#include "../common/platform.h" /* IWYU pragma: keep */
#include "../common/static_init.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct CmdLutElement {
uint8_t insert_len_extra_bits;
uint8_t copy_len_extra_bits;
int8_t distance_code;
uint8_t context;
uint16_t insert_len_offset;
uint16_t copy_len_offset;
} CmdLutElement;
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_NONE)
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
CmdLutElement kCmdLut[BROTLI_NUM_COMMAND_SYMBOLS];
#else
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderInitCmdLut(CmdLutElement* items);
BROTLI_INTERNAL extern BROTLI_MODEL("small")
CmdLutElement kCmdLut[BROTLI_NUM_COMMAND_SYMBOLS];
#endif
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_PREFIX_H_ */

View File

@@ -1,186 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "state.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "huffman.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#ifdef BROTLI_REPORTING
/* When BROTLI_REPORTING is defined extra reporting module have to be linked. */
void BrotliDecoderOnStart(const BrotliDecoderState* s);
void BrotliDecoderOnFinish(const BrotliDecoderState* s);
#define BROTLI_DECODER_ON_START(s) BrotliDecoderOnStart(s);
#define BROTLI_DECODER_ON_FINISH(s) BrotliDecoderOnFinish(s);
#else
#if !defined(BROTLI_DECODER_ON_START)
#define BROTLI_DECODER_ON_START(s) (void)(s);
#endif
#if !defined(BROTLI_DECODER_ON_FINISH)
#define BROTLI_DECODER_ON_FINISH(s) (void)(s);
#endif
#endif
BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
BROTLI_DECODER_ON_START(s);
if (!alloc_func) {
s->alloc_func = BrotliDefaultAllocFunc;
s->free_func = BrotliDefaultFreeFunc;
s->memory_manager_opaque = 0;
} else {
s->alloc_func = alloc_func;
s->free_func = free_func;
s->memory_manager_opaque = opaque;
}
s->error_code = 0; /* BROTLI_DECODER_NO_ERROR */
BrotliInitBitReader(&s->br);
s->state = BROTLI_STATE_UNINITED;
s->large_window = 0;
s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;
s->substate_uncompressed = BROTLI_STATE_UNCOMPRESSED_NONE;
s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE;
s->substate_read_block_length = BROTLI_STATE_READ_BLOCK_LENGTH_NONE;
s->buffer_length = 0;
s->loop_counter = 0;
s->pos = 0;
s->rb_roundtrips = 0;
s->partial_pos_out = 0;
s->used_input = 0;
s->block_type_trees = NULL;
s->block_len_trees = NULL;
s->ringbuffer = NULL;
s->ringbuffer_size = 0;
s->new_ringbuffer_size = 0;
s->ringbuffer_mask = 0;
s->context_map = NULL;
s->context_modes = NULL;
s->dist_context_map = NULL;
s->context_map_slice = NULL;
s->dist_context_map_slice = NULL;
s->literal_hgroup.codes = NULL;
s->literal_hgroup.htrees = NULL;
s->insert_copy_hgroup.codes = NULL;
s->insert_copy_hgroup.htrees = NULL;
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->is_last_metablock = 0;
s->is_uncompressed = 0;
s->is_metadata = 0;
s->should_wrap_ringbuffer = 0;
s->canny_ringbuffer_allocation = 1;
s->window_bits = 0;
s->max_distance = 0;
s->dist_rb[0] = 16;
s->dist_rb[1] = 15;
s->dist_rb[2] = 11;
s->dist_rb[3] = 4;
s->dist_rb_idx = 0;
s->block_type_trees = NULL;
s->block_len_trees = NULL;
s->mtf_upper_bound = 63;
s->compound_dictionary = NULL;
s->dictionary =
BrotliSharedDictionaryCreateInstance(alloc_func, free_func, opaque);
if (!s->dictionary) return BROTLI_FALSE;
s->metadata_start_func = NULL;
s->metadata_chunk_func = NULL;
s->metadata_callback_opaque = 0;
return BROTLI_TRUE;
}
void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s) {
s->meta_block_remaining_len = 0;
s->block_length[0] = BROTLI_BLOCK_SIZE_CAP;
s->block_length[1] = BROTLI_BLOCK_SIZE_CAP;
s->block_length[2] = BROTLI_BLOCK_SIZE_CAP;
s->num_block_types[0] = 1;
s->num_block_types[1] = 1;
s->num_block_types[2] = 1;
s->block_type_rb[0] = 1;
s->block_type_rb[1] = 0;
s->block_type_rb[2] = 1;
s->block_type_rb[3] = 0;
s->block_type_rb[4] = 1;
s->block_type_rb[5] = 0;
s->context_map = NULL;
s->context_modes = NULL;
s->dist_context_map = NULL;
s->context_map_slice = NULL;
s->literal_htree = NULL;
s->dist_context_map_slice = NULL;
s->dist_htree_index = 0;
s->context_lookup = NULL;
s->literal_hgroup.codes = NULL;
s->literal_hgroup.htrees = NULL;
s->insert_copy_hgroup.codes = NULL;
s->insert_copy_hgroup.htrees = NULL;
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
}
void BrotliDecoderStateCleanupAfterMetablock(BrotliDecoderState* s) {
BROTLI_DECODER_FREE(s, s->context_modes);
BROTLI_DECODER_FREE(s, s->context_map);
BROTLI_DECODER_FREE(s, s->dist_context_map);
BROTLI_DECODER_FREE(s, s->literal_hgroup.htrees);
BROTLI_DECODER_FREE(s, s->insert_copy_hgroup.htrees);
BROTLI_DECODER_FREE(s, s->distance_hgroup.htrees);
}
void BrotliDecoderStateCleanup(BrotliDecoderState* s) {
BrotliDecoderStateCleanupAfterMetablock(s);
BROTLI_DECODER_ON_FINISH(s);
BROTLI_DECODER_FREE(s, s->compound_dictionary);
BrotliSharedDictionaryDestroyInstance(s->dictionary);
s->dictionary = NULL;
BROTLI_DECODER_FREE(s, s->ringbuffer);
BROTLI_DECODER_FREE(s, s->block_type_trees);
}
BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s,
HuffmanTreeGroup* group, brotli_reg_t alphabet_size_max,
brotli_reg_t alphabet_size_limit, brotli_reg_t ntrees) {
/* 376 = 256 (1-st level table) + 4 + 7 + 15 + 31 + 63 (2-nd level mix-tables)
This number is discovered "unlimited" "enough" calculator; it is actually
a wee bigger than required in several cases (especially for alphabets with
less than 16 symbols). */
const size_t max_table_size = alphabet_size_limit + 376;
const size_t code_size = sizeof(HuffmanCode) * ntrees * max_table_size;
const size_t htree_size = sizeof(HuffmanCode*) * ntrees;
/* Pointer alignment is, hopefully, wider than sizeof(HuffmanCode). */
HuffmanCode** p = (HuffmanCode**)BROTLI_DECODER_ALLOC(s,
code_size + htree_size);
group->alphabet_size_max = (uint16_t)alphabet_size_max;
group->alphabet_size_limit = (uint16_t)alphabet_size_limit;
group->num_htrees = (uint16_t)ntrees;
group->htrees = p;
group->codes = p ? (HuffmanCode*)(&p[ntrees]) : NULL;
return !!p;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,396 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Brotli state for partial streaming decoding. */
#ifndef BROTLI_DEC_STATE_H_
#define BROTLI_DEC_STATE_H_
#include "../common/constants.h"
#include "../common/platform.h"
#include <brotli/shared_dictionary.h>
#include <brotli/decode.h>
#include "bit_reader.h"
#include "huffman.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Graphviz diagram that describes state transitions:
digraph States {
graph [compound=true]
concentrate=true
node [shape="box"]
UNINITED -> {LARGE_WINDOW_BITS -> INITIALIZE}
subgraph cluster_metablock_workflow {
style="rounded"
label=< <B>METABLOCK CYCLE</B> >
METABLOCK_BEGIN -> METABLOCK_HEADER
METABLOCK_HEADER:sw -> METADATA
METABLOCK_HEADER:s -> UNCOMPRESSED
METABLOCK_HEADER:se -> METABLOCK_DONE:ne
METADATA:s -> METABLOCK_DONE:w
UNCOMPRESSED:s -> METABLOCK_DONE:n
METABLOCK_DONE:e -> METABLOCK_BEGIN:e [constraint="false"]
}
INITIALIZE -> METABLOCK_BEGIN
METABLOCK_DONE -> DONE
subgraph cluster_compressed_metablock {
style="rounded"
label=< <B>COMPRESSED METABLOCK</B> >
subgraph cluster_command {
style="rounded"
label=< <B>HOT LOOP</B> >
_METABLOCK_DONE_PORT_ [shape=point style=invis]
{
// Set different shape for nodes returning from "compressed metablock".
node [shape=invhouse]; CMD_INNER CMD_POST_DECODE_LITERALS;
CMD_POST_WRAP_COPY; CMD_INNER_WRITE; CMD_POST_WRITE_1;
}
CMD_BEGIN -> CMD_INNER -> CMD_POST_DECODE_LITERALS -> CMD_POST_WRAP_COPY
// IO ("write") nodes are not in the hot loop!
CMD_INNER_WRITE [style=dashed]
CMD_INNER -> CMD_INNER_WRITE
CMD_POST_WRITE_1 [style=dashed]
CMD_POST_DECODE_LITERALS -> CMD_POST_WRITE_1
CMD_POST_WRITE_2 [style=dashed]
CMD_POST_WRAP_COPY -> CMD_POST_WRITE_2
CMD_POST_WRITE_1 -> CMD_BEGIN:s [constraint="false"]
CMD_INNER_WRITE -> {CMD_INNER CMD_POST_DECODE_LITERALS}
[constraint="false"]
CMD_BEGIN:ne -> CMD_POST_DECODE_LITERALS [constraint="false"]
CMD_POST_WRAP_COPY -> CMD_BEGIN [constraint="false"]
CMD_POST_DECODE_LITERALS -> CMD_BEGIN:ne [constraint="false"]
CMD_POST_WRITE_2 -> CMD_POST_WRAP_COPY [constraint="false"]
{rank=same; CMD_BEGIN; CMD_INNER; CMD_POST_DECODE_LITERALS;
CMD_POST_WRAP_COPY}
{rank=same; CMD_INNER_WRITE; CMD_POST_WRITE_1; CMD_POST_WRITE_2}
{CMD_INNER CMD_POST_DECODE_LITERALS CMD_POST_WRAP_COPY} ->
_METABLOCK_DONE_PORT_ [style=invis]
{CMD_INNER_WRITE CMD_POST_WRITE_1} -> _METABLOCK_DONE_PORT_
[constraint="false" style=invis]
}
BEFORE_COMPRESSED_METABLOCK_HEADER:s -> HUFFMAN_CODE_0:n
HUFFMAN_CODE_0 -> HUFFMAN_CODE_1 -> HUFFMAN_CODE_2 -> HUFFMAN_CODE_3
HUFFMAN_CODE_0 -> METABLOCK_HEADER_2 -> CONTEXT_MODES -> CONTEXT_MAP_1
CONTEXT_MAP_1 -> CONTEXT_MAP_2 -> TREE_GROUP
TREE_GROUP -> BEFORE_COMPRESSED_METABLOCK_BODY:e
BEFORE_COMPRESSED_METABLOCK_BODY:s -> CMD_BEGIN:n
HUFFMAN_CODE_3:e -> HUFFMAN_CODE_0:ne [constraint="false"]
{rank=same; HUFFMAN_CODE_0; HUFFMAN_CODE_1; HUFFMAN_CODE_2; HUFFMAN_CODE_3}
{rank=same; METABLOCK_HEADER_2; CONTEXT_MODES; CONTEXT_MAP_1; CONTEXT_MAP_2;
TREE_GROUP}
}
METABLOCK_HEADER:e -> BEFORE_COMPRESSED_METABLOCK_HEADER:n
_METABLOCK_DONE_PORT_ -> METABLOCK_DONE:se
[constraint="false" ltail=cluster_command]
UNINITED [shape=Mdiamond];
DONE [shape=Msquare];
}
*/
typedef enum {
BROTLI_STATE_UNINITED,
BROTLI_STATE_LARGE_WINDOW_BITS,
BROTLI_STATE_INITIALIZE,
BROTLI_STATE_METABLOCK_BEGIN,
BROTLI_STATE_METABLOCK_HEADER,
BROTLI_STATE_METABLOCK_HEADER_2,
BROTLI_STATE_CONTEXT_MODES,
BROTLI_STATE_COMMAND_BEGIN,
BROTLI_STATE_COMMAND_INNER,
BROTLI_STATE_COMMAND_POST_DECODE_LITERALS,
BROTLI_STATE_COMMAND_POST_WRAP_COPY,
BROTLI_STATE_UNCOMPRESSED,
BROTLI_STATE_METADATA,
BROTLI_STATE_COMMAND_INNER_WRITE,
BROTLI_STATE_METABLOCK_DONE,
BROTLI_STATE_COMMAND_POST_WRITE_1,
BROTLI_STATE_COMMAND_POST_WRITE_2,
BROTLI_STATE_BEFORE_COMPRESSED_METABLOCK_HEADER,
BROTLI_STATE_HUFFMAN_CODE_0,
BROTLI_STATE_HUFFMAN_CODE_1,
BROTLI_STATE_HUFFMAN_CODE_2,
BROTLI_STATE_HUFFMAN_CODE_3,
BROTLI_STATE_CONTEXT_MAP_1,
BROTLI_STATE_CONTEXT_MAP_2,
BROTLI_STATE_TREE_GROUP,
BROTLI_STATE_BEFORE_COMPRESSED_METABLOCK_BODY,
BROTLI_STATE_DONE
} BrotliRunningState;
typedef enum {
BROTLI_STATE_METABLOCK_HEADER_NONE,
BROTLI_STATE_METABLOCK_HEADER_EMPTY,
BROTLI_STATE_METABLOCK_HEADER_NIBBLES,
BROTLI_STATE_METABLOCK_HEADER_SIZE,
BROTLI_STATE_METABLOCK_HEADER_UNCOMPRESSED,
BROTLI_STATE_METABLOCK_HEADER_RESERVED,
BROTLI_STATE_METABLOCK_HEADER_BYTES,
BROTLI_STATE_METABLOCK_HEADER_METADATA
} BrotliRunningMetablockHeaderState;
typedef enum {
BROTLI_STATE_UNCOMPRESSED_NONE,
BROTLI_STATE_UNCOMPRESSED_WRITE
} BrotliRunningUncompressedState;
typedef enum {
BROTLI_STATE_TREE_GROUP_NONE,
BROTLI_STATE_TREE_GROUP_LOOP
} BrotliRunningTreeGroupState;
typedef enum {
BROTLI_STATE_CONTEXT_MAP_NONE,
BROTLI_STATE_CONTEXT_MAP_READ_PREFIX,
BROTLI_STATE_CONTEXT_MAP_HUFFMAN,
BROTLI_STATE_CONTEXT_MAP_DECODE,
BROTLI_STATE_CONTEXT_MAP_TRANSFORM
} BrotliRunningContextMapState;
typedef enum {
BROTLI_STATE_HUFFMAN_NONE,
BROTLI_STATE_HUFFMAN_SIMPLE_SIZE,
BROTLI_STATE_HUFFMAN_SIMPLE_READ,
BROTLI_STATE_HUFFMAN_SIMPLE_BUILD,
BROTLI_STATE_HUFFMAN_COMPLEX,
BROTLI_STATE_HUFFMAN_LENGTH_SYMBOLS
} BrotliRunningHuffmanState;
typedef enum {
BROTLI_STATE_DECODE_UINT8_NONE,
BROTLI_STATE_DECODE_UINT8_SHORT,
BROTLI_STATE_DECODE_UINT8_LONG
} BrotliRunningDecodeUint8State;
typedef enum {
BROTLI_STATE_READ_BLOCK_LENGTH_NONE,
BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
} BrotliRunningReadBlockLengthState;
/* BrotliDecoderState addon, used for Compound Dictionary functionality. */
typedef struct BrotliDecoderCompoundDictionary {
int num_chunks;
int total_size;
int br_index;
int br_offset;
int br_length;
int br_copied;
const uint8_t* chunks[16];
int chunk_offsets[16];
int block_bits;
uint8_t block_map[256];
} BrotliDecoderCompoundDictionary;
typedef struct BrotliMetablockHeaderArena {
BrotliRunningTreeGroupState substate_tree_group;
BrotliRunningContextMapState substate_context_map;
BrotliRunningHuffmanState substate_huffman;
brotli_reg_t sub_loop_counter;
brotli_reg_t repeat_code_len;
brotli_reg_t prev_code_len;
/* For ReadHuffmanCode. */
brotli_reg_t symbol;
brotli_reg_t repeat;
brotli_reg_t space;
/* Huffman table for "histograms". */
HuffmanCode table[32];
/* List of heads of symbol chains. */
uint16_t* symbol_lists;
/* Storage from symbol_lists. */
uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
BROTLI_NUM_COMMAND_SYMBOLS];
/* Tails of symbol chains. */
int next_symbol[32];
uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
/* Population counts for the code lengths. */
uint16_t code_length_histo[16];
/* TODO(eustas): +2 bytes padding */
/* For HuffmanTreeGroupDecode. */
int htree_index;
HuffmanCode* next;
/* For DecodeContextMap. */
brotli_reg_t context_index;
brotli_reg_t max_run_length_prefix;
brotli_reg_t code;
HuffmanCode context_map_table[BROTLI_HUFFMAN_MAX_SIZE_272];
} BrotliMetablockHeaderArena;
typedef struct BrotliMetablockBodyArena {
uint8_t dist_extra_bits[544];
brotli_reg_t dist_offset[544];
} BrotliMetablockBodyArena;
struct BrotliDecoderStateStruct {
BrotliRunningState state;
/* This counter is reused for several disjoint loops. */
int loop_counter;
BrotliBitReader br;
brotli_alloc_func alloc_func;
brotli_free_func free_func;
void* memory_manager_opaque;
/* Temporary storage for remaining input. Brotli stream format is designed in
a way, that 64 bits are enough to make progress in decoding. */
union {
uint64_t u64;
uint8_t u8[8];
} buffer;
brotli_reg_t buffer_length;
int pos;
int max_backward_distance;
int max_distance;
int ringbuffer_size;
int ringbuffer_mask;
int dist_rb_idx;
int dist_rb[4];
int error_code;
int meta_block_remaining_len;
uint8_t* ringbuffer;
uint8_t* ringbuffer_end;
HuffmanCode* htree_command;
const uint8_t* context_lookup;
uint8_t* context_map_slice;
uint8_t* dist_context_map_slice;
/* This ring buffer holds a few past copy distances that will be used by
some special distance codes. */
HuffmanTreeGroup literal_hgroup;
HuffmanTreeGroup insert_copy_hgroup;
HuffmanTreeGroup distance_hgroup;
HuffmanCode* block_type_trees;
HuffmanCode* block_len_trees;
/* This is true if the literal context map histogram type always matches the
block type. It is then not needed to keep the context (faster decoding). */
int trivial_literal_context;
/* Distance context is actual after command is decoded and before distance is
computed. After distance computation it is used as a temporary variable. */
int distance_context;
brotli_reg_t block_length[3];
brotli_reg_t block_length_index;
brotli_reg_t num_block_types[3];
brotli_reg_t block_type_rb[6];
brotli_reg_t distance_postfix_bits;
brotli_reg_t num_direct_distance_codes;
brotli_reg_t num_dist_htrees;
uint8_t* dist_context_map;
HuffmanCode* literal_htree;
/* For partial write operations. */
size_t rb_roundtrips; /* how many times we went around the ring-buffer */
size_t partial_pos_out; /* how much output to the user in total */
/* For InverseMoveToFrontTransform. */
brotli_reg_t mtf_upper_bound;
uint32_t mtf[64 + 1];
int copy_length;
int distance_code;
uint8_t dist_htree_index;
/* TODO(eustas): +3 bytes padding */
/* Less used attributes are at the end of this struct. */
brotli_decoder_metadata_start_func metadata_start_func;
brotli_decoder_metadata_chunk_func metadata_chunk_func;
void* metadata_callback_opaque;
/* For reporting. */
uint64_t used_input; /* how many bytes of input are consumed */
/* States inside function calls. */
BrotliRunningMetablockHeaderState substate_metablock_header;
BrotliRunningUncompressedState substate_uncompressed;
BrotliRunningDecodeUint8State substate_decode_uint8;
BrotliRunningReadBlockLengthState substate_read_block_length;
int new_ringbuffer_size;
/* TODO(eustas): +4 bytes padding */
unsigned int is_last_metablock : 1;
unsigned int is_uncompressed : 1;
unsigned int is_metadata : 1;
unsigned int should_wrap_ringbuffer : 1;
unsigned int canny_ringbuffer_allocation : 1;
unsigned int large_window : 1;
unsigned int window_bits : 6;
unsigned int size_nibbles : 8;
/* TODO(eustas): +12 bits padding */
brotli_reg_t num_literal_htrees;
uint8_t* context_map;
uint8_t* context_modes;
BrotliSharedDictionary* dictionary;
BrotliDecoderCompoundDictionary* compound_dictionary;
uint32_t trivial_literal_contexts[8]; /* 256 bits */
union {
BrotliMetablockHeaderArena header;
BrotliMetablockBodyArena body;
} arena;
};
typedef struct BrotliDecoderStateStruct BrotliDecoderStateInternal;
#define BrotliDecoderState BrotliDecoderStateInternal
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliDecoderStateCleanup(BrotliDecoderState* s);
BROTLI_INTERNAL void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s);
BROTLI_INTERNAL void BrotliDecoderStateCleanupAfterMetablock(
BrotliDecoderState* s);
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
BrotliDecoderState* s, HuffmanTreeGroup* group,
brotli_reg_t alphabet_size_max, brotli_reg_t alphabet_size_limit,
brotli_reg_t ntrees);
#define BROTLI_DECODER_ALLOC(S, L) S->alloc_func(S->memory_manager_opaque, L)
#define BROTLI_DECODER_FREE(S, X) { \
S->free_func(S->memory_manager_opaque, X); \
X = NULL; \
}
/* Literal/Command/Distance block size maximum; same as maximum metablock size;
used as block size when there is no block switching. */
#define BROTLI_BLOCK_SIZE_CAP (1U << 24)
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_STATE_H_ */

View File

@@ -1,53 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "static_init.h"
#include "../common/platform.h"
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/dictionary.h"
#include "prefix.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
static BROTLI_BOOL DoBrotliDecoderStaticInit(void) {
BROTLI_BOOL ok = BrotliDecoderInitCmdLut(kCmdLut);
if (!ok) return BROTLI_FALSE;
return BROTLI_TRUE;
}
#endif /* BROTLI_STATIC_INIT_NONE */
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_EARLY)
static BROTLI_BOOL kEarlyInitOk;
static __attribute__((constructor)) void BrotliDecoderStaticInitEarly(void) {
kEarlyInitOk = DoBrotliDecoderStaticInit();
}
#elif (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_LAZY)
static BROTLI_BOOL kLazyInitOk;
void BrotliDecoderLazyStaticInitInner(void) {
kLazyInitOk = DoBrotliDecoderStaticInit();
}
#endif /* BROTLI_STATIC_INIT_EARLY */
BROTLI_BOOL BrotliDecoderEnsureStaticInit(void) {
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_NONE)
return BROTLI_TRUE;
#elif (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_EARLY)
return kEarlyInitOk;
#else
return kLazyInitOk;
#endif
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,30 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Central point for static initialization. */
#ifndef THIRD_PARTY_BROTLI_DEC_STATIC_INIT_H_
#define THIRD_PARTY_BROTLI_DEC_STATIC_INIT_H_
#include "../common/platform.h"
#include "../common/static_init.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_LAZY)
BROTLI_INTERNAL void BrotliDecoderLazyStaticInitInner(void);
BROTLI_INTERNAL void BrotliDecoderLazyStaticInit(void);
#endif /* BROTLI_STATIC_INIT */
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderEnsureStaticInit(void);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif // THIRD_PARTY_BROTLI_DEC_STATIC_INIT_H_

View File

@@ -1,231 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#include "backward_references.h"
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/platform.h"
#include "command.h"
#include "compound_dictionary.h"
#include "encoder_dict.h"
#include "hash.h"
#include "params.h"
#include "quality.h" /* IWYU pragma: keep for inc */
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
size_t max_distance,
const int* dist_cache) {
if (distance <= max_distance) {
size_t distance_plus_3 = distance + 3;
size_t offset0 = distance_plus_3 - (size_t)dist_cache[0];
size_t offset1 = distance_plus_3 - (size_t)dist_cache[1];
if (distance == (size_t)dist_cache[0]) {
return 0;
} else if (distance == (size_t)dist_cache[1]) {
return 1;
} else if (offset0 < 7) {
return (0x9750468 >> (4 * offset0)) & 0xF;
} else if (offset1 < 7) {
return (0xFDB1ACE >> (4 * offset1)) & 0xF;
} else if (distance == (size_t)dist_cache[2]) {
return 2;
} else if (distance == (size_t)dist_cache[3]) {
return 3;
}
}
return distance + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
}
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
#define PREFIX() N
#define ENABLE_COMPOUND_DICTIONARY 0
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H3
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H4
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H5
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H6
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H40
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H41
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H42
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H54
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H35
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H55
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H65
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#if defined(BROTLI_MAX_SIMD_QUALITY)
#define HASHER() H58
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H68
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#endif
#undef ENABLE_COMPOUND_DICTIONARY
#undef PREFIX
#define PREFIX() D
#define ENABLE_COMPOUND_DICTIONARY 1
#define HASHER() H5
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H6
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H40
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H41
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H42
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H55
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H65
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#if defined(BROTLI_MAX_SIMD_QUALITY)
#define HASHER() H58
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#define HASHER() H68
/* NOLINTNEXTLINE(build/include) */
#include "backward_references_inc.h"
#undef HASHER
#endif
#undef ENABLE_COMPOUND_DICTIONARY
#undef PREFIX
#undef EXPORT_FN
#undef FN
#undef CAT
#undef EXPAND_CAT
void BrotliCreateBackwardReferences(size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals) {
if (params->dictionary.compound.num_chunks != 0) {
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesDH ## N(num_bytes, \
position, ringbuffer, ringbuffer_mask, \
literal_context_lut, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
return;
CASE_(5)
CASE_(6)
#if defined(BROTLI_MAX_SIMD_QUALITY)
CASE_(58)
CASE_(68)
#endif
CASE_(40)
CASE_(41)
CASE_(42)
CASE_(55)
CASE_(65)
#undef CASE_
default:
BROTLI_DCHECK(BROTLI_FALSE);
break;
}
}
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesNH ## N(num_bytes, \
position, ringbuffer, ringbuffer_mask, \
literal_context_lut, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
return;
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:
BROTLI_DCHECK(BROTLI_FALSE);
break;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,36 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
#include "../common/context.h"
#include "../common/platform.h"
#include "command.h"
#include "hash.h"
#include "params.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */

View File

@@ -1,92 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
#include "../common/context.h"
#include "../common/platform.h"
#include "command.h"
#include "hash.h"
#include "memory.h"
#include "params.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
typedef struct ZopfliNode {
/* Best length to get up to this byte (not including this byte itself)
highest 7 bit is used to reconstruct the length code. */
uint32_t length;
/* Distance associated with the length. */
uint32_t distance;
/* Number of literal inserts before this copy; highest 5 bits contain
distance short code + 1 (or zero if no short code). */
uint32_t dcode_insert_length;
/* This union holds information used by dynamic-programming. During forward
pass |cost| it used to store the goal function. When node is processed its
|cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|next| is assigned the offset to next node on the path. */
union {
/* Smallest cost to get to this byte from the beginning, as found so far. */
float cost;
/* Offset to the next node on the path. Equals to command_length() of the
next node on the path. For last node equals to BROTLI_UINT32_MAX */
uint32_t next;
/* Node position that provides next distance for distance cache. */
uint32_t shortcut;
} u;
} ZopfliNode;
BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
/* Computes the shortest path of commands from position to at most
position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the
length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following
"ZopfliNode array invariant":
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
ContextLut literal_context_lut, const BrotliEncoderParams* params,
const int* dist_cache, Hasher* hasher, ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes,
int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
Command* commands, size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */

View File

@@ -1,32 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to estimate the bit cost of Huffman trees. */
#ifndef BROTLI_ENC_BIT_COST_H_
#define BROTLI_ENC_BIT_COST_H_
#include "../common/platform.h"
#include "histogram.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
BROTLI_INTERNAL double BrotliBitsEntropy(
const uint32_t* population, size_t size);
BROTLI_INTERNAL double BrotliPopulationCostLiteral(
const HistogramLiteral* histogram);
BROTLI_INTERNAL double BrotliPopulationCostCommand(
const HistogramCommand* histogram);
BROTLI_INTERNAL double BrotliPopulationCostDistance(
const HistogramDistance* histogram);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_BIT_COST_H_ */

View File

@@ -1,83 +0,0 @@
/* Copyright 2014 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to convert brotli-related data structures into the
brotli bit stream. The functions here operate under
assumption that there is enough space in the storage, i.e., there are
no out-of-range checks anywhere.
These functions do bit addressing into a byte array. The byte array
is called "storage" and the index to the bit is called storage_ix
in function arguments. */
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
#include "../common/context.h"
#include "../common/platform.h"
#include "command.h"
#include "entropy_encode.h"
#include "memory.h"
#include "metablock.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* All Store functions here will use a storage_ix, which is always the bit
position for the current storage. */
BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage);
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
HuffmanTree* tree, const uint32_t* histogram, const size_t histogram_total,
const size_t max_bits, uint8_t* depth, uint16_t* bits, size_t* storage_ix,
uint8_t* storage);
/* REQUIRES: length > 0 */
/* REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlock(MemoryManager* m,
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
const BrotliEncoderParams* params, ContextType literal_context_mode,
const Command* commands, size_t n_commands, const MetaBlockSplit* mb,
size_t* storage_ix, uint8_t* storage);
/* Stores the meta-block without doing any block splitting, just collects
one histogram per block category and uses that for entropy coding.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlockTrivial(MemoryManager* m,
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
const Command* commands, size_t n_commands,
size_t* storage_ix, uint8_t* storage);
/* Same as above, but uses static prefix codes for histograms with a only a few
symbols, and uses static code length prefix codes for all other histograms.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlockFast(MemoryManager* m,
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
const Command* commands, size_t n_commands,
size_t* storage_ix, uint8_t* storage);
/* This is for storing uncompressed blocks (simple raw storage of
bytes-as-bytes).
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
BROTLI_BOOL is_final_block, const uint8_t* BROTLI_RESTRICT input,
size_t position, size_t mask, size_t len,
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_BROTLI_BIT_STREAM_H_ */

View File

@@ -1,28 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "command.h"
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
const uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES] = {
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26,
34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594};
const uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES] = {
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24};
const uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES] = {
2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18,
22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118};
const uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES] = {
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24};
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,189 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* This class models a sequence of literals and a backward reference copy. */
#ifndef BROTLI_ENC_COMMAND_H_
#define BROTLI_ENC_COMMAND_H_
#include "../common/constants.h"
#include "../common/platform.h"
#include "fast_log.h"
#include "params.h"
#include "prefix.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES];
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES];
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES];
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES];
static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
if (insertlen < 6) {
return (uint16_t)insertlen;
} else if (insertlen < 130) {
uint32_t nbits = Log2FloorNonZero(insertlen - 2) - 1u;
return (uint16_t)((nbits << 1) + ((insertlen - 2) >> nbits) + 2);
} else if (insertlen < 2114) {
return (uint16_t)(Log2FloorNonZero(insertlen - 66) + 10);
} else if (insertlen < 6210) {
return 21u;
} else if (insertlen < 22594) {
return 22u;
} else {
return 23u;
}
}
static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
if (copylen < 10) {
return (uint16_t)(copylen - 2);
} else if (copylen < 134) {
uint32_t nbits = Log2FloorNonZero(copylen - 6) - 1u;
return (uint16_t)((nbits << 1) + ((copylen - 6) >> nbits) + 4);
} else if (copylen < 2118) {
return (uint16_t)(Log2FloorNonZero(copylen - 70) + 12);
} else {
return 23u;
}
}
static BROTLI_INLINE uint16_t CombineLengthCodes(
uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
uint16_t bits64 =
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u));
if (use_last_distance && inscode < 8u && copycode < 16u) {
return (copycode < 8u) ? bits64 : (bits64 | 64u);
} else {
/* Specification: 5 Encoding of ... (last table) */
/* offset = 2 * index, where index is in range [0..8] */
uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u));
/* All values in specification are K * 64,
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
All values in D require only 2 bits to encode.
Magic constant is shifted 6 bits left, to avoid final multiplication. */
offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u);
return (uint16_t)(offset | bits64);
}
}
static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
BROTLI_BOOL use_last_distance,
uint16_t* code) {
uint16_t inscode = GetInsertLengthCode(insertlen);
uint16_t copycode = GetCopyLengthCode(copylen);
*code = CombineLengthCodes(inscode, copycode, use_last_distance);
}
static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
return kBrotliInsBase[inscode];
}
static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
return kBrotliInsExtra[inscode];
}
static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
return kBrotliCopyBase[copycode];
}
static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
return kBrotliCopyExtra[copycode];
}
typedef struct Command {
uint32_t insert_len_;
/* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */
uint32_t copy_len_;
/* Stores distance extra bits. */
uint32_t dist_extra_;
uint16_t cmd_prefix_;
/* Stores distance code in low 10 bits
and number of extra bits in high 6 bits. */
uint16_t dist_prefix_;
} Command;
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
static BROTLI_INLINE void InitCommand(Command* self,
const BrotliDistanceParams* dist, size_t insertlen,
size_t copylen, int copylen_code_delta, size_t distance_code) {
/* Don't rely on signed int representation, use honest casts. */
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = (uint32_t)(copylen | (delta << 25));
/* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */
PrefixEncodeCopyDistance(
distance_code, dist->num_direct_distance_codes,
dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_);
GetLengthCode(
insertlen, (size_t)((int)copylen + copylen_code_delta),
TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_);
}
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = 4 << 25;
self->dist_extra_ = 0;
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
}
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
const Command* self, const BrotliDistanceParams* dist) {
if ((self->dist_prefix_ & 0x3FFu) <
BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) {
return self->dist_prefix_ & 0x3FFu;
} else {
uint32_t dcode = self->dist_prefix_ & 0x3FFu;
uint32_t nbits = self->dist_prefix_ >> 10;
uint32_t extra = self->dist_extra_;
uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U;
uint32_t hcode = (dcode - dist->num_direct_distance_codes -
BROTLI_NUM_DISTANCE_SHORT_CODES) >>
dist->distance_postfix_bits;
uint32_t lcode = (dcode - dist->num_direct_distance_codes -
BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask;
uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U;
return ((offset + extra) << dist->distance_postfix_bits) + lcode +
dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES;
}
}
static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
uint32_t r = self->cmd_prefix_ >> 6;
uint32_t c = self->cmd_prefix_ & 7;
if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) {
return c;
}
return 3;
}
static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
return self->copy_len_ & 0x1FFFFFF;
}
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
uint32_t modifier = self->copy_len_ >> 25;
int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1)));
return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta);
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_COMMAND_H_ */

View File

@@ -1,205 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "compound_dictionary.h"
#include "../common/platform.h"
#include <brotli/shared_dictionary.h>
#include "memory.h"
static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
const uint8_t* source, size_t source_size, uint32_t bucket_bits,
uint32_t slot_bits, uint32_t hash_bits, uint16_t bucket_limit) {
/* Step 1: create "bloated" hasher. */
uint32_t num_slots = 1u << slot_bits;
uint32_t num_buckets = 1u << bucket_bits;
uint32_t hash_shift = 64u - bucket_bits;
uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
uint32_t slot_mask = num_slots - 1;
size_t alloc_size = (sizeof(uint32_t) << slot_bits) +
(sizeof(uint32_t) << slot_bits) +
(sizeof(uint16_t) << bucket_bits) +
(sizeof(uint32_t) << bucket_bits) +
(sizeof(uint32_t) * source_size);
uint8_t* flat = NULL;
PreparedDictionary* result = NULL;
uint16_t* num = NULL;
uint32_t* bucket_heads = NULL;
uint32_t* next_bucket = NULL;
uint32_t* slot_offsets = NULL;
uint16_t* heads = NULL;
uint32_t* items = NULL;
uint8_t** source_ref = NULL;
uint32_t i;
uint32_t* slot_size = NULL;
uint32_t* slot_limit = NULL;
uint32_t total_items = 0;
if (slot_bits > 16) return NULL;
if (slot_bits > bucket_bits) return NULL;
if (bucket_bits - slot_bits >= 16) return NULL;
flat = BROTLI_ALLOC(m, uint8_t, alloc_size);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(flat)) return NULL;
slot_size = (uint32_t*)flat;
slot_limit = (uint32_t*)(&slot_size[num_slots]);
num = (uint16_t*)(&slot_limit[num_slots]);
bucket_heads = (uint32_t*)(&num[num_buckets]);
next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
memset(num, 0, num_buckets * sizeof(num[0]));
/* TODO(eustas): apply custom "store" order. */
for (i = 0; i + 7 < source_size; ++i) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
kPreparedDictionaryHashMul64Long;
const uint32_t key = (uint32_t)(h >> hash_shift);
uint16_t count = num[key];
next_bucket[i] = (count == 0) ? ((uint32_t)(-1)) : bucket_heads[key];
bucket_heads[key] = i;
count++;
if (count > bucket_limit) count = bucket_limit;
num[key] = count;
}
/* Step 2: find slot limits. */
for (i = 0; i < num_slots; ++i) {
BROTLI_BOOL overflow = BROTLI_FALSE;
slot_limit[i] = bucket_limit;
while (BROTLI_TRUE) {
uint32_t limit = slot_limit[i];
size_t j;
uint32_t count = 0;
overflow = BROTLI_FALSE;
for (j = i; j < num_buckets; j += num_slots) {
uint32_t size = num[j];
/* Last chain may span behind 64K limit; overflow happens only if
we are about to use 0xFFFF+ as item offset. */
if (count >= 0xFFFF) {
overflow = BROTLI_TRUE;
break;
}
if (size > limit) size = limit;
count += size;
}
if (!overflow) {
slot_size[i] = count;
total_items += count;
break;
}
slot_limit[i]--;
}
}
/* Step 3: transfer data to "slim" hasher. */
alloc_size = sizeof(PreparedDictionary) + (sizeof(uint32_t) << slot_bits) +
(sizeof(uint16_t) << bucket_bits) + (sizeof(uint32_t) * total_items) +
sizeof(uint8_t*);
result = (PreparedDictionary*)BROTLI_ALLOC(m, uint8_t, alloc_size);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(result)) {
BROTLI_FREE(m, flat);
return NULL;
}
slot_offsets = (uint32_t*)(&result[1]);
heads = (uint16_t*)(&slot_offsets[num_slots]);
items = (uint32_t*)(&heads[num_buckets]);
source_ref = (uint8_t**)(&items[total_items]);
result->magic = kLeanPreparedDictionaryMagic;
result->num_items = total_items;
result->source_size = (uint32_t)source_size;
result->hash_bits = hash_bits;
result->bucket_bits = bucket_bits;
result->slot_bits = slot_bits;
BROTLI_UNALIGNED_STORE_PTR(source_ref, source);
total_items = 0;
for (i = 0; i < num_slots; ++i) {
slot_offsets[i] = total_items;
total_items += slot_size[i];
slot_size[i] = 0;
}
for (i = 0; i < num_buckets; ++i) {
uint32_t slot = i & slot_mask;
uint32_t count = num[i];
uint32_t pos;
size_t j;
size_t cursor = slot_size[slot];
if (count > slot_limit[slot]) count = slot_limit[slot];
if (count == 0) {
heads[i] = 0xFFFF;
continue;
}
heads[i] = (uint16_t)cursor;
cursor += slot_offsets[slot];
slot_size[slot] += count;
pos = bucket_heads[i];
for (j = 0; j < count; j++) {
items[cursor++] = pos;
pos = next_bucket[pos];
}
items[cursor - 1] |= 0x80000000;
}
BROTLI_FREE(m, flat);
return result;
}
PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
const uint8_t* source, size_t source_size) {
uint32_t bucket_bits = 17;
uint32_t slot_bits = 7;
uint32_t hash_bits = 40;
uint16_t bucket_limit = 32;
size_t volume = 16u << bucket_bits;
/* Tune parameters to fit dictionary size. */
while (volume < source_size && bucket_bits < 22) {
bucket_bits++;
slot_bits++;
volume <<= 1;
}
return CreatePreparedDictionaryWithParams(m,
source, source_size, bucket_bits, slot_bits, hash_bits, bucket_limit);
}
void DestroyPreparedDictionary(MemoryManager* m,
PreparedDictionary* dictionary) {
if (!dictionary) return;
BROTLI_FREE(m, dictionary);
}
BROTLI_BOOL AttachPreparedDictionary(
CompoundDictionary* compound, const PreparedDictionary* dictionary) {
size_t length = 0;
size_t index = 0;
if (compound->num_chunks == SHARED_BROTLI_MAX_COMPOUND_DICTS) {
return BROTLI_FALSE;
}
if (!dictionary) return BROTLI_FALSE;
length = dictionary->source_size;
index = compound->num_chunks;
compound->total_size += length;
compound->chunks[index] = dictionary;
compound->chunk_offsets[index + 1] = compound->total_size;
{
uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]);
uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << dictionary->slot_bits]);
uint32_t* items = (uint32_t*)(&heads[(size_t)1u << dictionary->bucket_bits]);
const void* tail = (void*)&items[dictionary->num_items];
if (dictionary->magic == kPreparedDictionaryMagic) {
compound->chunk_source[index] = (const uint8_t*)tail;
} else {
/* dictionary->magic == kLeanPreparedDictionaryMagic */
compound->chunk_source[index] =
(const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
}
}
compound->num_chunks++;
return BROTLI_TRUE;
}

View File

@@ -1,71 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#ifndef BROTLI_ENC_PREPARED_DICTIONARY_H_
#define BROTLI_ENC_PREPARED_DICTIONARY_H_
#include "../common/platform.h"
#include <brotli/shared_dictionary.h>
#include "memory.h"
/* "Fat" prepared dictionary, could be cooked outside of C implementation,
* e.g. on Java side. LZ77 data is copied inside PreparedDictionary struct. */
static const uint32_t kPreparedDictionaryMagic = 0xDEBCEDE0;
static const uint32_t kSharedDictionaryMagic = 0xDEBCEDE1;
static const uint32_t kManagedDictionaryMagic = 0xDEBCEDE2;
/* "Lean" prepared dictionary. LZ77 data is referenced. It is the responsibility
* of caller of "prepare dictionary" to keep the LZ77 data while prepared
* dictionary is in use. */
static const uint32_t kLeanPreparedDictionaryMagic = 0xDEBCEDE3;
static const uint64_t kPreparedDictionaryHashMul64Long =
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
typedef struct PreparedDictionary {
uint32_t magic;
uint32_t num_items;
uint32_t source_size;
uint32_t hash_bits;
uint32_t bucket_bits;
uint32_t slot_bits;
/* --- Dynamic size members --- */
/* uint32_t slot_offsets[1 << slot_bits]; */
/* uint16_t heads[1 << bucket_bits]; */
/* uint32_t items[variable]; */
/* [maybe] uint8_t* source_ref, depending on magic. */
/* [maybe] uint8_t source[source_size], depending on magic. */
} PreparedDictionary;
BROTLI_INTERNAL PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
const uint8_t* source, size_t source_size);
BROTLI_INTERNAL void DestroyPreparedDictionary(MemoryManager* m,
PreparedDictionary* dictionary);
typedef struct CompoundDictionary {
/* LZ77 prefix, compound dictionary */
size_t num_chunks;
size_t total_size;
/* Client instances. */
const PreparedDictionary* chunks[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
const uint8_t* chunk_source[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
size_t chunk_offsets[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
size_t num_prepared_instances_;
/* Owned instances. */
PreparedDictionary* prepared_instances_[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
} CompoundDictionary;
BROTLI_INTERNAL BROTLI_BOOL AttachPreparedDictionary(
CompoundDictionary* compound, const PreparedDictionary* dictionary);
#endif /* BROTLI_ENC_PREPARED_DICTIONARY */

View File

@@ -1,647 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function for fast encoding of an input fragment, independently from the input
history. This function uses two-pass processing: in the first pass we save
the found backward matches and literal bytes into a buffer, and in the
second pass we emit them into the bit stream using prefix codes built based
on the actual command and literal byte histograms. */
#include "compress_fragment_two_pass.h"
#include "../common/constants.h"
#include "../common/platform.h"
#include "bit_cost.h"
#include "brotli_bit_stream.h"
#include "entropy_encode.h"
#include "fast_log.h"
#include "find_match_length.h"
#include "hash_base.h"
#include "write_bits.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define MAX_DISTANCE (long)BROTLI_MAX_BACKWARD_LIMIT(18)
static BROTLI_INLINE uint32_t Hash(const uint8_t* p,
size_t shift, size_t length) {
const uint64_t h =
(BROTLI_UNALIGNED_LOAD64LE(p) << ((8 - length) * 8)) * kHashMul32;
return (uint32_t)(h >> shift);
}
static BROTLI_INLINE uint32_t HashBytesAtOffset(uint64_t v, size_t offset,
size_t shift, size_t length) {
BROTLI_DCHECK(offset <= 8 - length);
{
const uint64_t h = ((v >> (8 * offset)) << ((8 - length) * 8)) * kHashMul32;
return (uint32_t)(h >> shift);
}
}
static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2,
size_t length) {
if (BrotliUnalignedRead32(p1) == BrotliUnalignedRead32(p2)) {
if (length == 4) return BROTLI_TRUE;
return TO_BROTLI_BOOL(p1[4] == p2[4] && p1[5] == p2[5]);
}
return BROTLI_FALSE;
}
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
"bits" based on "histogram" and stores it into the bit stream. */
static void BuildAndStoreCommandPrefixCode(BrotliTwoPassArena* s,
size_t* storage_ix,
uint8_t* storage) {
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
/* TODO(eustas): initialize once. */
memset(s->tmp_depth, 0, sizeof(s->tmp_depth));
BrotliCreateHuffmanTree(s->cmd_histo, 64, 15, s->tmp_tree, s->cmd_depth);
BrotliCreateHuffmanTree(&s->cmd_histo[64], 64, 14, s->tmp_tree,
&s->cmd_depth[64]);
/* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in
the full alphabet. This looks complicated, but having the symbols
in this order in the command bits saves a few branches in the Emit*
functions. */
memcpy(s->tmp_depth, s->cmd_depth + 24, 24);
memcpy(s->tmp_depth + 24, s->cmd_depth, 8);
memcpy(s->tmp_depth + 32, s->cmd_depth + 48, 8);
memcpy(s->tmp_depth + 40, s->cmd_depth + 8, 8);
memcpy(s->tmp_depth + 48, s->cmd_depth + 56, 8);
memcpy(s->tmp_depth + 56, s->cmd_depth + 16, 8);
BrotliConvertBitDepthsToSymbols(s->tmp_depth, 64, s->tmp_bits);
memcpy(s->cmd_bits, s->tmp_bits + 24, 16);
memcpy(s->cmd_bits + 8, s->tmp_bits + 40, 16);
memcpy(s->cmd_bits + 16, s->tmp_bits + 56, 16);
memcpy(s->cmd_bits + 24, s->tmp_bits, 48);
memcpy(s->cmd_bits + 48, s->tmp_bits + 32, 16);
memcpy(s->cmd_bits + 56, s->tmp_bits + 48, 16);
BrotliConvertBitDepthsToSymbols(&s->cmd_depth[64], 64, &s->cmd_bits[64]);
{
/* Create the bit length array for the full command alphabet. */
size_t i;
memset(s->tmp_depth, 0, 64); /* only 64 first values were used */
memcpy(s->tmp_depth, s->cmd_depth + 24, 8);
memcpy(s->tmp_depth + 64, s->cmd_depth + 32, 8);
memcpy(s->tmp_depth + 128, s->cmd_depth + 40, 8);
memcpy(s->tmp_depth + 192, s->cmd_depth + 48, 8);
memcpy(s->tmp_depth + 384, s->cmd_depth + 56, 8);
for (i = 0; i < 8; ++i) {
s->tmp_depth[128 + 8 * i] = s->cmd_depth[i];
s->tmp_depth[256 + 8 * i] = s->cmd_depth[8 + i];
s->tmp_depth[448 + 8 * i] = s->cmd_depth[16 + i];
}
BrotliStoreHuffmanTree(s->tmp_depth, BROTLI_NUM_COMMAND_SYMBOLS,
s->tmp_tree, storage_ix, storage);
}
BrotliStoreHuffmanTree(&s->cmd_depth[64], 64, s->tmp_tree, storage_ix,
storage);
}
static BROTLI_INLINE void EmitInsertLen(
uint32_t insertlen, uint32_t** commands) {
if (insertlen < 6) {
**commands = insertlen;
} else if (insertlen < 130) {
const uint32_t tail = insertlen - 2;
const uint32_t nbits = Log2FloorNonZero(tail) - 1u;
const uint32_t prefix = tail >> nbits;
const uint32_t inscode = (nbits << 1) + prefix + 2;
const uint32_t extra = tail - (prefix << nbits);
**commands = inscode | (extra << 8);
} else if (insertlen < 2114) {
const uint32_t tail = insertlen - 66;
const uint32_t nbits = Log2FloorNonZero(tail);
const uint32_t code = nbits + 10;
const uint32_t extra = tail - (1u << nbits);
**commands = code | (extra << 8);
} else if (insertlen < 6210) {
const uint32_t extra = insertlen - 2114;
**commands = 21 | (extra << 8);
} else if (insertlen < 22594) {
const uint32_t extra = insertlen - 6210;
**commands = 22 | (extra << 8);
} else {
const uint32_t extra = insertlen - 22594;
**commands = 23 | (extra << 8);
}
++(*commands);
}
static BROTLI_INLINE void EmitCopyLen(size_t copylen, uint32_t** commands) {
if (copylen < 10) {
**commands = (uint32_t)(copylen + 38);
} else if (copylen < 134) {
const size_t tail = copylen - 6;
const size_t nbits = Log2FloorNonZero(tail) - 1;
const size_t prefix = tail >> nbits;
const size_t code = (nbits << 1) + prefix + 44;
const size_t extra = tail - (prefix << nbits);
**commands = (uint32_t)(code | (extra << 8));
} else if (copylen < 2118) {
const size_t tail = copylen - 70;
const size_t nbits = Log2FloorNonZero(tail);
const size_t code = nbits + 52;
const size_t extra = tail - ((size_t)1 << nbits);
**commands = (uint32_t)(code | (extra << 8));
} else {
const size_t extra = copylen - 2118;
**commands = (uint32_t)(63 | (extra << 8));
}
++(*commands);
}
static BROTLI_INLINE void EmitCopyLenLastDistance(
size_t copylen, uint32_t** commands) {
if (copylen < 12) {
**commands = (uint32_t)(copylen + 20);
++(*commands);
} else if (copylen < 72) {
const size_t tail = copylen - 8;
const size_t nbits = Log2FloorNonZero(tail) - 1;
const size_t prefix = tail >> nbits;
const size_t code = (nbits << 1) + prefix + 28;
const size_t extra = tail - (prefix << nbits);
**commands = (uint32_t)(code | (extra << 8));
++(*commands);
} else if (copylen < 136) {
const size_t tail = copylen - 8;
const size_t code = (tail >> 5) + 54;
const size_t extra = tail & 31;
**commands = (uint32_t)(code | (extra << 8));
++(*commands);
**commands = 64;
++(*commands);
} else if (copylen < 2120) {
const size_t tail = copylen - 72;
const size_t nbits = Log2FloorNonZero(tail);
const size_t code = nbits + 52;
const size_t extra = tail - ((size_t)1 << nbits);
**commands = (uint32_t)(code | (extra << 8));
++(*commands);
**commands = 64;
++(*commands);
} else {
const size_t extra = copylen - 2120;
**commands = (uint32_t)(63 | (extra << 8));
++(*commands);
**commands = 64;
++(*commands);
}
}
static BROTLI_INLINE void EmitDistance(uint32_t distance, uint32_t** commands) {
uint32_t d = distance + 3;
uint32_t nbits = Log2FloorNonZero(d) - 1;
const uint32_t prefix = (d >> nbits) & 1;
const uint32_t offset = (2 + prefix) << nbits;
const uint32_t distcode = 2 * (nbits - 1) + prefix + 80;
uint32_t extra = d - offset;
**commands = distcode | (extra << 8);
++(*commands);
}
/* REQUIRES: len <= 1 << 24. */
static void BrotliStoreMetaBlockHeader(
size_t len, BROTLI_BOOL is_uncompressed, size_t* storage_ix,
uint8_t* storage) {
size_t nibbles = 6;
/* ISLAST */
BrotliWriteBits(1, 0, storage_ix, storage);
if (len <= (1U << 16)) {
nibbles = 4;
} else if (len <= (1U << 20)) {
nibbles = 5;
}
BrotliWriteBits(2, nibbles - 4, storage_ix, storage);
BrotliWriteBits(nibbles * 4, len - 1, storage_ix, storage);
/* ISUNCOMPRESSED */
BrotliWriteBits(1, (uint64_t)is_uncompressed, storage_ix, storage);
}
static BROTLI_INLINE void CreateCommands(const uint8_t* input,
size_t block_size, size_t input_size, const uint8_t* base_ip, int* table,
size_t table_bits, size_t min_match,
uint8_t** literals, uint32_t** commands) {
/* "ip" is the input pointer. */
const uint8_t* ip = input;
const size_t shift = 64u - table_bits;
const uint8_t* ip_end = input + block_size;
/* "next_emit" is a pointer to the first byte that is not covered by a
previous copy. Bytes between "next_emit" and the start of the next copy or
the end of the input will be emitted as literal bytes. */
const uint8_t* next_emit = input;
int last_distance = -1;
const size_t kInputMarginBytes = BROTLI_WINDOW_GAP;
if (BROTLI_PREDICT_TRUE(block_size >= kInputMarginBytes)) {
/* For the last block, we need to keep a 16 bytes margin so that we can be
sure that all distances are at most window size - 16.
For all other blocks, we only need to keep a margin of 5 bytes so that
we don't go over the block size with a copy. */
const size_t len_limit = BROTLI_MIN(size_t, block_size - min_match,
input_size - kInputMarginBytes);
const uint8_t* ip_limit = input + len_limit;
uint32_t next_hash;
for (next_hash = Hash(++ip, shift, min_match); ; ) {
/* Step 1: Scan forward in the input looking for a 6-byte-long match.
If we get close to exhausting the input then goto emit_remainder.
Heuristic match skipping: If 32 bytes are scanned with no matches
found, start looking only at every other byte. If 32 more bytes are
scanned, look at every third byte, etc.. When a match is found,
immediately go back to looking at every byte. This is a small loss
(~5% performance, ~0.1% density) for compressible data due to more
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
win since the compressor quickly "realizes" the data is incompressible
and doesn't bother looking for matches everywhere.
The "skip" variable keeps track of how many bytes there are since the
last match; dividing it by 32 (ie. right-shifting by five) gives the
number of bytes to move ahead for each iteration. */
uint32_t skip = 32;
const uint8_t* next_ip = ip;
const uint8_t* candidate;
BROTLI_DCHECK(next_emit < ip);
trawl:
do {
uint32_t hash = next_hash;
uint32_t bytes_between_hash_lookups = skip++ >> 5;
ip = next_ip;
BROTLI_DCHECK(hash == Hash(ip, shift, min_match));
next_ip = ip + bytes_between_hash_lookups;
if (BROTLI_PREDICT_FALSE(next_ip > ip_limit)) {
goto emit_remainder;
}
next_hash = Hash(next_ip, shift, min_match);
candidate = ip - last_distance;
if (IsMatch(ip, candidate, min_match)) {
if (BROTLI_PREDICT_TRUE(candidate < ip)) {
table[hash] = (int)(ip - base_ip);
break;
}
}
candidate = base_ip + table[hash];
BROTLI_DCHECK(candidate >= base_ip);
BROTLI_DCHECK(candidate < ip);
table[hash] = (int)(ip - base_ip);
} while (BROTLI_PREDICT_TRUE(!IsMatch(ip, candidate, min_match)));
/* Check copy distance. If candidate is not feasible, continue search.
Checking is done outside of hot loop to reduce overhead. */
if (ip - candidate > MAX_DISTANCE) goto trawl;
/* Step 2: Emit the found match together with the literal bytes from
"next_emit", and then see if we can find a next match immediately
afterwards. Repeat until we find no match for the input
without emitting some literal bytes. */
{
/* We have a 6-byte match at ip, and we need to emit bytes in
[next_emit, ip). */
const uint8_t* base = ip;
size_t matched = min_match + FindMatchLengthWithLimit(
candidate + min_match, ip + min_match,
(size_t)(ip_end - ip) - min_match);
int distance = (int)(base - candidate); /* > 0 */
int insert = (int)(base - next_emit);
ip += matched;
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
EmitInsertLen((uint32_t)insert, commands);
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
(int)(next_emit - base_ip), insert, 2));
memcpy(*literals, next_emit, (size_t)insert);
*literals += insert;
if (distance == last_distance) {
**commands = 64;
++(*commands);
} else {
EmitDistance((uint32_t)distance, commands);
last_distance = distance;
}
EmitCopyLenLastDistance(matched, commands);
BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
"[CompressFragment] pos = %d insert = %d copy = %d\n"
"[CompressFragment] pos = %d distance = %d\n",
(int)(base - base_ip), (int)distance,
(int)(base - base_ip) + 2, 0, (int)matched - 2,
(int)(base - base_ip) + 2, (int)distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
goto emit_remainder;
}
{
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes;
uint32_t cur_hash;
uint32_t prev_hash;
if (min_match == 4) {
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
cur_hash = HashBytesAtOffset(input_bytes, 3, shift, min_match);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 3);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 2);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 1);
} else {
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 5);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift, min_match);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 2);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 1);
}
candidate = base_ip + table[cur_hash];
table[cur_hash] = (int)(ip - base_ip);
}
}
while (ip - candidate <= MAX_DISTANCE &&
IsMatch(ip, candidate, min_match)) {
/* We have a 6-byte match at ip, and no need to emit any
literal bytes prior to ip. */
const uint8_t* base = ip;
size_t matched = min_match + FindMatchLengthWithLimit(
candidate + min_match, ip + min_match,
(size_t)(ip_end - ip) - min_match);
ip += matched;
last_distance = (int)(base - candidate); /* > 0 */
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
EmitCopyLen(matched, commands);
EmitDistance((uint32_t)last_distance, commands);
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
"[CompressFragment] pos = %d distance = %d\n",
(int)(base - base_ip), 0, (int)matched,
(int)(base - base_ip), (int)last_distance));
next_emit = ip;
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
goto emit_remainder;
}
{
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes;
uint32_t cur_hash;
uint32_t prev_hash;
if (min_match == 4) {
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
cur_hash = HashBytesAtOffset(input_bytes, 3, shift, min_match);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 3);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 2);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 1);
} else {
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 5);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift, min_match);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 2);
prev_hash = HashBytesAtOffset(input_bytes, 1, shift, min_match);
table[prev_hash] = (int)(ip - base_ip - 1);
}
candidate = base_ip + table[cur_hash];
table[cur_hash] = (int)(ip - base_ip);
}
}
next_hash = Hash(++ip, shift, min_match);
}
}
emit_remainder:
BROTLI_DCHECK(next_emit <= ip_end);
/* Emit the remaining bytes as literals. */
if (next_emit < ip_end) {
const uint32_t insert = (uint32_t)(ip_end - next_emit);
EmitInsertLen(insert, commands);
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
(int)(next_emit - base_ip), insert, 2));
memcpy(*literals, next_emit, insert);
*literals += insert;
}
}
static void StoreCommands(BrotliTwoPassArena* s,
const uint8_t* literals, const size_t num_literals,
const uint32_t* commands, const size_t num_commands,
size_t* storage_ix, uint8_t* storage) {
static const BROTLI_MODEL("small") uint32_t kNumExtraBits[128] = {
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5,
6, 7, 8, 9, 10, 12, 14, 24, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24,
};
static const BROTLI_MODEL("small") uint32_t kInsertOffset[24] = {
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26,
34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594,
};
size_t i;
memset(s->lit_histo, 0, sizeof(s->lit_histo));
/* TODO(eustas): is that necessary? */
memset(s->cmd_depth, 0, sizeof(s->cmd_depth));
/* TODO(eustas): is that necessary? */
memset(s->cmd_bits, 0, sizeof(s->cmd_bits));
memset(s->cmd_histo, 0, sizeof(s->cmd_histo));
for (i = 0; i < num_literals; ++i) {
++s->lit_histo[literals[i]];
}
BrotliBuildAndStoreHuffmanTreeFast(s->tmp_tree, s->lit_histo, num_literals,
/* max_bits = */ 8, s->lit_depth,
s->lit_bits, storage_ix, storage);
for (i = 0; i < num_commands; ++i) {
const uint32_t code = commands[i] & 0xFF;
BROTLI_DCHECK(code < 128);
++s->cmd_histo[code];
}
s->cmd_histo[1] += 1;
s->cmd_histo[2] += 1;
s->cmd_histo[64] += 1;
s->cmd_histo[84] += 1;
BuildAndStoreCommandPrefixCode(s, storage_ix, storage);
for (i = 0; i < num_commands; ++i) {
const uint32_t cmd = commands[i];
const uint32_t code = cmd & 0xFF;
const uint32_t extra = cmd >> 8;
BROTLI_DCHECK(code < 128);
BrotliWriteBits(s->cmd_depth[code], s->cmd_bits[code], storage_ix, storage);
BrotliWriteBits(kNumExtraBits[code], extra, storage_ix, storage);
if (code < 24) {
const uint32_t insert = kInsertOffset[code] + extra;
uint32_t j;
for (j = 0; j < insert; ++j) {
const uint8_t lit = *literals;
BrotliWriteBits(s->lit_depth[lit], s->lit_bits[lit], storage_ix,
storage);
++literals;
}
}
}
}
/* Acceptable loss for uncompressible speedup is 2% */
#define MIN_RATIO 0.98
#define SAMPLE_RATE 43
static BROTLI_BOOL ShouldCompress(BrotliTwoPassArena* s,
const uint8_t* input, size_t input_size, size_t num_literals) {
double corpus_size = (double)input_size;
if ((double)num_literals < MIN_RATIO * corpus_size) {
return BROTLI_TRUE;
} else {
const double max_total_bit_cost = corpus_size * 8 * MIN_RATIO / SAMPLE_RATE;
size_t i;
memset(s->lit_histo, 0, sizeof(s->lit_histo));
for (i = 0; i < input_size; i += SAMPLE_RATE) {
++s->lit_histo[input[i]];
}
return TO_BROTLI_BOOL(
BrotliBitsEntropy(s->lit_histo, 256) < max_total_bit_cost);
}
}
static void RewindBitPosition(const size_t new_storage_ix,
size_t* storage_ix, uint8_t* storage) {
const size_t bitpos = new_storage_ix & 7;
const size_t mask = (1u << bitpos) - 1;
storage[new_storage_ix >> 3] &= (uint8_t)mask;
*storage_ix = new_storage_ix;
}
static void EmitUncompressedMetaBlock(const uint8_t* input, size_t input_size,
size_t* storage_ix, uint8_t* storage) {
BrotliStoreMetaBlockHeader(input_size, 1, storage_ix, storage);
*storage_ix = (*storage_ix + 7u) & ~7u;
memcpy(&storage[*storage_ix >> 3], input, input_size);
*storage_ix += input_size << 3;
storage[*storage_ix >> 3] = 0;
}
static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl(
BrotliTwoPassArena* s, const uint8_t* input, size_t input_size,
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
int* table, size_t table_bits, size_t min_match,
size_t* storage_ix, uint8_t* storage) {
/* Save the start of the first block for position and distance computations.
*/
const uint8_t* base_ip = input;
BROTLI_UNUSED(is_last);
while (input_size > 0) {
size_t block_size =
BROTLI_MIN(size_t, input_size, kCompressFragmentTwoPassBlockSize);
uint32_t* commands = command_buf;
uint8_t* literals = literal_buf;
size_t num_literals;
CreateCommands(input, block_size, input_size, base_ip, table,
table_bits, min_match, &literals, &commands);
num_literals = (size_t)(literals - literal_buf);
if (ShouldCompress(s, input, block_size, num_literals)) {
const size_t num_commands = (size_t)(commands - command_buf);
BrotliStoreMetaBlockHeader(block_size, 0, storage_ix, storage);
/* No block splits, no contexts. */
BrotliWriteBits(13, 0, storage_ix, storage);
StoreCommands(s, literal_buf, num_literals, command_buf, num_commands,
storage_ix, storage);
} else {
/* Since we did not find many backward references and the entropy of
the data is close to 8 bits, we can simply emit an uncompressed block.
This makes compression speed of uncompressible data about 3x faster. */
EmitUncompressedMetaBlock(input, block_size, storage_ix, storage);
}
input += block_size;
input_size -= block_size;
}
}
#define FOR_TABLE_BITS_(X) \
X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17)
#define BAKE_METHOD_PARAM_(B) \
static BROTLI_NOINLINE void BrotliCompressFragmentTwoPassImpl ## B( \
BrotliTwoPassArena* s, const uint8_t* input, size_t input_size, \
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, \
int* table, size_t* storage_ix, uint8_t* storage) { \
size_t min_match = (B <= 15) ? 4 : 6; \
BrotliCompressFragmentTwoPassImpl(s, input, input_size, is_last, command_buf,\
literal_buf, table, B, min_match, storage_ix, storage); \
}
FOR_TABLE_BITS_(BAKE_METHOD_PARAM_)
#undef BAKE_METHOD_PARAM_
void BrotliCompressFragmentTwoPass(
BrotliTwoPassArena* s, const uint8_t* input, size_t input_size,
BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf,
int* table, size_t table_size, size_t* storage_ix, uint8_t* storage) {
const size_t initial_storage_ix = *storage_ix;
const size_t table_bits = Log2FloorNonZero(table_size);
switch (table_bits) {
#define CASE_(B) \
case B: \
BrotliCompressFragmentTwoPassImpl ## B( \
s, input, input_size, is_last, command_buf, \
literal_buf, table, storage_ix, storage); \
break;
FOR_TABLE_BITS_(CASE_)
#undef CASE_
default: BROTLI_DCHECK(0); break;
}
/* If output is larger than single uncompressed block, rewrite it. */
if (*storage_ix - initial_storage_ix > 31 + (input_size << 3)) {
RewindBitPosition(initial_storage_ix, storage_ix, storage);
EmitUncompressedMetaBlock(input, input_size, storage_ix, storage);
}
if (is_last) {
BrotliWriteBits(1, 1, storage_ix, storage); /* islast */
BrotliWriteBits(1, 1, storage_ix, storage); /* isempty */
*storage_ix = (*storage_ix + 7u) & ~7u;
}
}
#undef FOR_TABLE_BITS_
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,145 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Hash table on the 4-byte prefixes of static dictionary words. */
#include "dictionary_hash.h"
#include "../common/platform.h" /* IWYU pragma: keep */
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/dictionary.h"
#include "hash_base.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
BROTLI_BOOL BROTLI_COLD BrotliEncoderInitDictionaryHash(
const BrotliDictionary* dict, uint16_t* words, uint8_t* lengths) {
size_t global_idx = 0;
size_t len;
size_t i;
static const uint8_t frozen_idx[1688] = {0, 0, 8, 164, 32, 56, 31, 191, 36, 4,
128, 81, 68, 132, 145, 129, 0, 0, 0, 28, 0, 8, 1, 1, 64, 3, 1, 0, 0, 0, 0, 0, 4,
64, 1, 2, 128, 0, 132, 49, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 36, 152,
0, 0, 0, 0, 128, 8, 0, 0, 128, 0, 0, 8, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8,
0, 0, 0, 1, 0, 64, 133, 0, 32, 0, 0, 128, 1, 0, 0, 0, 0, 4, 4, 4, 32, 16, 130,
0, 128, 8, 0, 0, 0, 0, 0, 64, 0, 64, 0, 160, 0, 148, 53, 0, 0, 0, 0, 0, 128, 0,
130, 0, 0, 0, 8, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 32, 1, 32, 129, 0, 12, 0,
1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8,
0, 0, 2, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2, 66, 128, 0, 0, 16, 0, 0, 0, 0, 64, 1, 6,
128, 8, 0, 192, 24, 32, 0, 0, 8, 4, 128, 128, 2, 160, 0, 160, 0, 64, 0, 0, 2, 0,
0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 32, 0, 66, 0, 2, 0,
4, 0, 8, 0, 2, 0, 0, 33, 8, 0, 0, 0, 8, 0, 128, 162, 4, 128, 0, 2, 33, 0, 160,
0, 8, 0, 64, 0, 160, 0, 129, 4, 0, 0, 32, 0, 0, 32, 0, 2, 0, 0, 0, 0, 0, 0, 128,
0, 0, 0, 0, 0, 64, 10, 0, 0, 0, 0, 32, 64, 0, 0, 0, 0, 0, 16, 0, 16, 16, 0, 0,
80, 2, 0, 0, 0, 0, 8, 0, 0, 16, 0, 8, 0, 0, 0, 8, 64, 128, 0, 0, 0, 8, 208, 0,
0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 32, 0, 8, 0, 128, 0, 0, 0, 1, 0, 0, 0,
16, 8, 1, 136, 0, 0, 36, 0, 64, 9, 0, 1, 32, 8, 0, 64, 64, 131, 16, 224, 32, 4,
0, 4, 5, 160, 0, 131, 0, 4, 96, 0, 0, 184, 192, 0, 177, 205, 96, 0, 0, 0, 0, 2,
0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 128, 0, 0, 8, 0, 0, 0, 0, 1, 4, 0, 1,
0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 64, 69, 0, 0, 8, 2, 66, 32, 64, 0, 0, 0,
0, 0, 1, 0, 128, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 16, 0, 0, 4, 128, 64,
0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 8, 0, 0, 130, 16, 64, 128, 2, 64, 0, 0, 0, 128,
2, 192, 64, 0, 65, 0, 0, 0, 16, 0, 0, 0, 32, 4, 2, 2, 76, 0, 0, 0, 4, 72, 52,
131, 44, 76, 0, 0, 0, 0, 64, 1, 16, 148, 4, 0, 16, 10, 64, 0, 2, 0, 1, 0, 128,
64, 68, 0, 0, 0, 0, 0, 64, 144, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 3, 64, 0, 0, 0, 0,
1, 128, 0, 0, 32, 66, 0, 0, 0, 40, 0, 18, 0, 0, 0, 0, 0, 33, 0, 0, 32, 0, 0, 32,
0, 128, 4, 64, 145, 140, 0, 0, 0, 128, 0, 2, 0, 0, 20, 0, 80, 38, 0, 0, 32, 0,
32, 64, 4, 4, 0, 4, 0, 0, 0, 129, 4, 0, 0, 144, 17, 32, 130, 16, 132, 24, 134,
0, 0, 64, 2, 5, 50, 8, 194, 33, 1, 68, 117, 1, 8, 32, 161, 54, 0, 130, 34, 0, 0,
0, 64, 128, 0, 0, 2, 0, 0, 0, 0, 32, 1, 0, 0, 0, 3, 14, 0, 0, 0, 0, 0, 16, 4, 0,
0, 0, 0, 0, 0, 0, 0, 96, 1, 24, 18, 0, 1, 128, 24, 0, 64, 0, 4, 0, 16, 128, 0,
64, 0, 0, 0, 64, 0, 8, 0, 0, 0, 0, 0, 66, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 16, 0, 64, 2, 0, 0, 0, 0, 6, 0, 8, 8, 2, 0, 64, 0, 0, 0, 0, 128, 2,
2, 12, 64, 0, 64, 0, 8, 0, 128, 32, 0, 0, 10, 0, 0, 32, 0, 128, 32, 33, 8, 136,
0, 96, 64, 0, 0, 0, 0, 0, 64, 4, 16, 4, 8, 0, 0, 0, 16, 0, 2, 0, 0, 1, 128, 0,
64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 16, 0, 4, 0, 8, 0, 0, 0, 0, 0,
20, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 136, 0, 0, 0, 0, 0, 8, 0,
0, 0, 0, 0, 2, 0, 0, 0, 64, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0,
0, 0, 4, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 2, 128, 0, 0, 0, 8, 2, 0, 0, 128, 0, 16, 2, 0, 0, 4, 0, 32, 0, 0, 1,
4, 64, 64, 0, 4, 0, 1, 0, 16, 0, 32, 68, 4, 4, 65, 10, 0, 20, 37, 18, 1, 148, 0,
32, 128, 3, 8, 0, 64, 0, 0, 0, 0, 0, 0, 4, 0, 16, 1, 128, 0, 0, 0, 128, 16, 0,
0, 0, 0, 1, 128, 0, 0, 128, 64, 128, 64, 0, 130, 0, 164, 8, 0, 0, 1, 64, 128, 0,
18, 0, 2, 150, 0, 8, 0, 0, 64, 0, 81, 0, 0, 16, 128, 2, 8, 36, 32, 129, 4, 144,
13, 0, 0, 3, 8, 1, 0, 2, 0, 0, 64, 0, 5, 0, 1, 34, 1, 32, 2, 16, 128, 128, 128,
0, 0, 0, 2, 0, 4, 18, 8, 12, 34, 32, 192, 6, 64, 224, 33, 0, 0, 137, 72, 64, 0,
24, 8, 128, 128, 0, 16, 0, 32, 128, 128, 132, 8, 0, 0, 16, 0, 64, 0, 0, 4, 0, 0,
16, 0, 4, 128, 64, 0, 0, 1, 0, 4, 64, 32, 144, 130, 2, 128, 0, 192, 0, 64, 82,
64, 1, 32, 128, 128, 2, 0, 84, 0, 32, 0, 44, 24, 72, 80, 32, 16, 0, 0, 44, 16,
96, 64, 1, 72, 131, 0, 0, 0, 16, 0, 0, 165, 0, 129, 2, 49, 48, 64, 64, 12, 64,
176, 64, 84, 8, 128, 20, 64, 213, 136, 104, 1, 41, 15, 83, 170, 0, 0, 41, 1, 64,
64, 0, 193, 64, 64, 8, 0, 128, 0, 0, 64, 8, 64, 8, 1, 16, 0, 8, 0, 0, 2, 1, 128,
28, 84, 141, 97, 0, 0, 68, 0, 0, 129, 8, 0, 16, 8, 32, 0, 64, 0, 0, 0, 24, 0, 0,
0, 192, 0, 8, 128, 0, 0, 0, 0, 0, 64, 0, 1, 0, 0, 0, 0, 40, 1, 128, 64, 0, 4, 2,
32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 32, 8, 0, 32, 0, 0, 0, 16, 17, 0,
2, 4, 0, 0, 33, 128, 2, 0, 0, 0, 0, 129, 0, 2, 0, 0, 0, 36, 0, 32, 2, 0, 0, 0,
0, 0, 0, 32, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 32, 64, 0, 0, 0, 0, 0, 0,
32, 0, 0, 32, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 16, 0, 0, 0, 0, 0, 0, 0,
1, 0, 136, 0, 0, 24, 192, 128, 3, 0, 17, 18, 2, 0, 66, 0, 4, 24, 0, 9, 208, 167,
0, 144, 20, 64, 0, 130, 64, 0, 2, 16, 136, 8, 74, 32, 0, 168, 0, 65, 32, 8, 12,
1, 3, 1, 64, 180, 3, 0, 64, 0, 8, 0, 0, 32, 65, 0, 4, 16, 4, 16, 68, 32, 64, 36,
32, 24, 33, 1, 128, 0, 0, 8, 0, 32, 64, 81, 0, 1, 10, 19, 8, 0, 0, 4, 5, 144, 0,
0, 8, 128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 80, 1, 0, 0,
33, 0, 32, 66, 4, 2, 0, 1, 43, 2, 0, 0, 4, 32, 16, 0, 64, 0, 3, 32, 0, 2, 64,
64, 116, 0, 65, 52, 64, 0, 17, 64, 192, 96, 8, 10, 8, 2, 4, 0, 17, 64, 0, 4, 0,
0, 4, 128, 0, 0, 9, 0, 0, 130, 2, 0, 192, 0, 48, 128, 64, 0, 96, 0, 64, 0, 1,
16, 32, 0, 1, 32, 6, 128, 2, 32, 0, 12, 0, 0, 48, 32, 8, 0, 0, 128, 0, 18, 0,
0, 28, 24, 41, 16, 5, 32, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8, 0, 0, 0, 0, 16, 128,
0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0};
memset(lengths, 0, BROTLI_ENC_NUM_HASH_BUCKETS);
for (len = BROTLI_MAX_DICTIONARY_WORD_LENGTH;
len >= BROTLI_MIN_DICTIONARY_WORD_LENGTH; --len) {
size_t length_lt_8 = len < 8 ? 1 : 0;
size_t n = 1u << dict->size_bits_by_length[len];
const uint8_t* dict_words = dict->data + dict->offsets_by_length[len];
for (i = 0; i < n; ++i) {
size_t j = n - 1 - i;
const uint8_t* word = dict_words + len * j;
const uint32_t key = Hash14(word);
size_t idx = (key << 1) + length_lt_8;
if ((lengths[idx] & 0x80) == 0) {
BROTLI_BOOL is_final = TO_BROTLI_BOOL(frozen_idx[global_idx / 8] &
(1u << (global_idx % 8)));
words[idx] = (uint16_t)j;
lengths[idx] = (uint8_t)(len + (is_final ? 0x80 : 0));
}
global_idx++;
}
}
for (i = 0; i < BROTLI_ENC_NUM_HASH_BUCKETS; ++i) {
lengths[i] &= 0x7F;
}
return BROTLI_TRUE;
}
BROTLI_MODEL("small")
uint16_t kStaticDictionaryHashWords[BROTLI_ENC_NUM_HASH_BUCKETS];
BROTLI_MODEL("small")
uint8_t kStaticDictionaryHashLengths[BROTLI_ENC_NUM_HASH_BUCKETS];
#else /* BROTLI_STATIC_INIT */
/* Embed kStaticDictionaryHashWords and kStaticDictionaryHashLengths. */
#include "dictionary_hash_inc.h"
#endif /* BROTLI_STATIC_INIT */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,45 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Hash table on the 4-byte prefixes of static dictionary words. */
#ifndef BROTLI_ENC_DICTIONARY_HASH_H_
#define BROTLI_ENC_DICTIONARY_HASH_H_
#include "../common/platform.h"
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/dictionary.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Bucket is (Hash14 * 2 + length_lt_8); in other words we reserve 2 buckets
for each hash - one for shorter words and one for longer words. */
#define BROTLI_ENC_NUM_HASH_BUCKETS 32768
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
BROTLI_BOOL BROTLI_INTERNAL BrotliEncoderInitDictionaryHash(
const BrotliDictionary* dictionary, uint16_t* words, uint8_t* lengths);
BROTLI_INTERNAL extern BROTLI_MODEL("small") uint16_t
kStaticDictionaryHashWords[BROTLI_ENC_NUM_HASH_BUCKETS];
BROTLI_INTERNAL extern BROTLI_MODEL("small") uint8_t
kStaticDictionaryHashLengths[BROTLI_ENC_NUM_HASH_BUCKETS];
#else
BROTLI_INTERNAL extern const BROTLI_MODEL("small") uint16_t
kStaticDictionaryHashWords[BROTLI_ENC_NUM_HASH_BUCKETS];
BROTLI_INTERNAL extern const BROTLI_MODEL("small") uint8_t
kStaticDictionaryHashLengths[BROTLI_ENC_NUM_HASH_BUCKETS];
#endif
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_DICTIONARY_HASH_H_ */

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,642 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "encoder_dict.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include <brotli/shared_dictionary.h>
#include "../common/shared_dictionary_internal.h"
#include "../common/transform.h"
#include <brotli/encode.h>
#include "compound_dictionary.h"
#include "dictionary_hash.h"
#include "hash_base.h"
#include "hash.h"
#include "memory.h"
#include "quality.h"
#include "static_dict_lut.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define NUM_HASH_BITS 15u
#define NUM_HASH_BUCKETS (1u << NUM_HASH_BITS)
static void BrotliTrieInit(BrotliTrie* trie) {
trie->pool_capacity = 0;
trie->pool_size = 0;
trie->pool = 0;
/* Set up the root node */
trie->root.single = 0;
trie->root.len_ = 0;
trie->root.idx_ = 0;
trie->root.sub = 0;
}
static void BrotliTrieFree(MemoryManager* m, BrotliTrie* trie) {
BrotliFree(m, trie->pool);
}
/* Initializes to RFC 7932 static dictionary / transforms. */
static void InitEncoderDictionary(BrotliEncoderDictionary* dict) {
dict->words = BrotliGetDictionary();
dict->num_transforms = (uint32_t)BrotliGetTransforms()->num_transforms;
dict->hash_table_words = kStaticDictionaryHashWords;
dict->hash_table_lengths = kStaticDictionaryHashLengths;
dict->buckets = kStaticDictionaryBuckets;
dict->dict_words = kStaticDictionaryWords;
dict->cutoffTransformsCount = kCutoffTransformsCount;
dict->cutoffTransforms = kCutoffTransforms;
dict->parent = 0;
dict->hash_table_data_words_ = 0;
dict->hash_table_data_lengths_ = 0;
dict->buckets_alloc_size_ = 0;
dict->buckets_data_ = 0;
dict->dict_words_alloc_size_ = 0;
dict->dict_words_data_ = 0;
dict->words_instance_ = 0;
dict->has_words_heavy = BROTLI_FALSE;
BrotliTrieInit(&dict->trie);
}
static void BrotliDestroyEncoderDictionary(MemoryManager* m,
BrotliEncoderDictionary* dict) {
BrotliFree(m, dict->hash_table_data_words_);
BrotliFree(m, dict->hash_table_data_lengths_);
BrotliFree(m, dict->buckets_data_);
BrotliFree(m, dict->dict_words_data_);
BrotliFree(m, dict->words_instance_);
BrotliTrieFree(m, &dict->trie);
}
#if defined(BROTLI_EXPERIMENTAL)
/* Word length must be at least 4 bytes */
static uint32_t Hash(const uint8_t* data, int bits) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - bits);
}
/* Theoretical max possible word size after transform */
#define kTransformedBufferSize \
(256 + 256 + SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH)
/* To be safe buffer must have at least kTransformedBufferSize */
static void TransformedDictionaryWord(uint32_t word_idx, int len, int transform,
const BrotliTransforms* transforms,
const BrotliEncoderDictionary* dict,
uint8_t* buffer, size_t* size) {
const uint8_t* dict_word = &dict->words->data[
dict->words->offsets_by_length[len] + (uint32_t)len * word_idx];
*size = (size_t)BrotliTransformDictionaryWord(buffer, dict_word, len,
transforms, transform);
}
static DictWord MakeDictWord(uint8_t len, uint8_t transform, uint16_t idx) {
DictWord result;
result.len = len;
result.transform = transform;
result.idx = idx;
return result;
}
static uint32_t BrotliTrieAlloc(MemoryManager* m, size_t num, BrotliTrie* trie,
BrotliTrieNode** keep) {
uint32_t result;
uint32_t keep_index = 0;
if (keep && *keep != &trie->root) {
/* Optional node to keep, since address may change after re-allocating */
keep_index = (uint32_t)(*keep - trie->pool);
}
if (trie->pool_size == 0) {
/* Have a placeholder node in the front. We do not want the result to be 0,
it must be at least 1, 0 represents "null pointer" */
trie->pool_size = 1;
}
BROTLI_ENSURE_CAPACITY(m, BrotliTrieNode, trie->pool, trie->pool_capacity,
trie->pool_size + num);
if (BROTLI_IS_OOM(m)) return 0;
/* Init the new nodes to empty */
memset(trie->pool + trie->pool_size, 0, sizeof(*trie->pool) * num);
result = (uint32_t)trie->pool_size;
trie->pool_size += num;
if (keep && *keep != &trie->root) {
*keep = trie->pool + keep_index;
}
return result;
}
/**
* len and idx: payload for last node
* word, size: the string
* index: position in the string
*/
static BROTLI_BOOL BrotliTrieNodeAdd(MemoryManager* m, uint8_t len,
uint32_t idx, const uint8_t* word, size_t size, int index,
BrotliTrieNode* node, BrotliTrie* trie) {
BrotliTrieNode* child = 0;
uint8_t c;
if ((size_t)index == size) {
if (!node->len_ || idx < node->idx_) {
node->len_ = len;
node->idx_ = idx;
}
return BROTLI_TRUE;
}
c = word[index];
if (node->single && c != node->c) {
BrotliTrieNode old = trie->pool[node->sub];
uint32_t new_nodes = BrotliTrieAlloc(m, 32, trie, &node);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
node->single = 0;
node->sub = new_nodes;
trie->pool[node->sub + (node->c >> 4)].sub = new_nodes + 16;
trie->pool[trie->pool[node->sub + (node->c >> 4)].sub + (node->c & 15)] =
old;
}
if (!node->sub) {
uint32_t new_node = BrotliTrieAlloc(m, 1, trie, &node);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
node->single = 1;
node->c = c;
node->sub = new_node;
}
if (node->single) {
child = &trie->pool[node->sub];
} else {
if (!trie->pool[node->sub + (c >> 4)].sub) {
uint32_t new_nodes = BrotliTrieAlloc(m, 16, trie, &node);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
trie->pool[node->sub + (c >> 4)].sub = new_nodes;
}
child = &trie->pool[trie->pool[node->sub + (c >> 4)].sub + (c & 15)];
}
return BrotliTrieNodeAdd(m, len, idx, word, size, index + 1, child, trie);
}
static BROTLI_BOOL BrotliTrieAdd(MemoryManager* m, uint8_t len, uint32_t idx,
const uint8_t* word, size_t size, BrotliTrie* trie) {
return BrotliTrieNodeAdd(m, len, idx, word, size, 0, &trie->root, trie);
}
const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
const BrotliTrieNode* node, uint8_t c) {
BrotliTrieNode* temp_node;
if (node->single) {
if (node->c == c) return &trie->pool[node->sub];
return 0;
}
if (!node->sub) return 0;
temp_node = &trie->pool[node->sub + (c >> 4)];
if (!temp_node->sub) return 0;
return &trie->pool[temp_node->sub + (c & 15)];
}
static const BrotliTrieNode* BrotliTrieFind(const BrotliTrie* trie,
const uint8_t* word, size_t size) {
const BrotliTrieNode* node = &trie->root;
size_t i;
for (i = 0; i < size; i++) {
node = BrotliTrieSub(trie, node, word[i]);
if (!node) return 0;
}
return node;
}
static BROTLI_BOOL BuildDictionaryLut(MemoryManager* m,
const BrotliTransforms* transforms,
BrotliEncoderDictionary* dict) {
uint32_t i;
DictWord* dict_words;
uint16_t* buckets;
DictWord** words_by_hash;
size_t* words_by_hash_size;
size_t* words_by_hash_capacity;
BrotliTrie dedup;
uint8_t word[kTransformedBufferSize];
size_t word_size;
size_t total = 0;
uint8_t l;
uint16_t idx;
BrotliTrieInit(&dedup);
words_by_hash = (DictWord**)BrotliAllocate(m,
sizeof(*words_by_hash) * NUM_HASH_BUCKETS);
words_by_hash_size = (size_t*)BrotliAllocate(m,
sizeof(*words_by_hash_size) * NUM_HASH_BUCKETS);
words_by_hash_capacity = (size_t*)BrotliAllocate(m,
sizeof(*words_by_hash_capacity) * NUM_HASH_BUCKETS);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
memset(words_by_hash, 0, sizeof(*words_by_hash) * NUM_HASH_BUCKETS);
memset(words_by_hash_size, 0, sizeof(*words_by_hash_size) * NUM_HASH_BUCKETS);
memset(words_by_hash_capacity, 0,
sizeof(*words_by_hash_capacity) * NUM_HASH_BUCKETS);
if (transforms->num_transforms > 0) {
for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
uint16_t n = dict->words->size_bits_by_length[l] ?
(uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u;
for (idx = 0; idx < n; ++idx) {
uint32_t key;
/* First transform (usually identity) */
TransformedDictionaryWord(idx, l, 0, transforms, dict, word,
&word_size);
/* Cannot hash words smaller than 4 bytes */
if (word_size < 4) {
/* Break instead of continue, all next words of this length will have
same length after transform */
break;
}
if (!BrotliTrieAdd(m, 0, idx, word, word_size, &dedup)) {
return BROTLI_FALSE;
}
key = Hash(word, NUM_HASH_BITS);
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, 0, idx));
++total;
}
}
}
/* These LUT transforms only supported if no custom transforms. This is
ok, we will use the heavy trie instead. */
if (transforms == BrotliGetTransforms()) {
for (l = SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH;
l <= SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH; ++l) {
uint16_t n = dict->words->size_bits_by_length[l] ?
(uint16_t)(1 << dict->words->size_bits_by_length[l]) : 0u;
for (idx = 0; idx < n; ++idx) {
int k;
BROTLI_BOOL is_ascii = BROTLI_TRUE;
size_t offset = dict->words->offsets_by_length[l] + (size_t)l * idx;
const uint8_t* data = &dict->words->data[offset];
for (k = 0; k < l; ++k) {
if (data[k] >= 128) is_ascii = BROTLI_FALSE;
}
if (data[0] < 128) {
int transform = 9; /* {empty, uppercase first, empty} */
uint32_t ix = idx + (uint32_t)transform * n;
const BrotliTrieNode* it;
TransformedDictionaryWord(idx, l, transform, transforms,
dict, word, &word_size);
it = BrotliTrieFind(&dedup, word, word_size);
if (!it || it->idx_ > ix) {
uint32_t key = Hash(word, NUM_HASH_BITS);
if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) {
return BROTLI_FALSE;
}
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_FIRST, idx));
++total;
}
}
if (is_ascii) {
int transform = 44; /* {empty, uppercase all, empty} */
uint32_t ix = idx + (uint32_t)transform * n;
const BrotliTrieNode* it;
TransformedDictionaryWord(idx, l, transform, transforms,
dict, word, &word_size);
it = BrotliTrieFind(&dedup, word, word_size);
if (!it || it->idx_ > ix) {
uint32_t key = Hash(word, NUM_HASH_BITS);
if (!BrotliTrieAdd(m, 0, ix, word, word_size, &dedup)) {
return BROTLI_FALSE;
}
BROTLI_ENSURE_CAPACITY_APPEND(m, DictWord, words_by_hash[key],
words_by_hash_capacity[key], words_by_hash_size[key],
MakeDictWord(l, BROTLI_TRANSFORM_UPPERCASE_ALL, idx));
++total;
}
}
}
}
}
dict_words = (DictWord*)BrotliAllocate(m,
sizeof(*dict->dict_words) * (total + 1));
buckets = (uint16_t*)BrotliAllocate(m,
sizeof(*dict->buckets) * NUM_HASH_BUCKETS);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
dict->dict_words_alloc_size_ = total + 1;
dict->dict_words = dict->dict_words_data_ = dict_words;
dict->buckets_alloc_size_ = NUM_HASH_BUCKETS;
dict->buckets = dict->buckets_data_ = buckets;
/* Unused; makes offsets start from 1. */
dict_words[0] = MakeDictWord(0, 0, 0);
total = 1;
for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
size_t num_words = words_by_hash_size[i];
if (num_words > 0) {
buckets[i] = (uint16_t)(total);
memcpy(&dict_words[total], &words_by_hash[i][0],
sizeof(dict_words[0]) * num_words);
total += num_words;
dict_words[total - 1].len |= 0x80;
} else {
buckets[i] = 0;
}
}
for (i = 0; i < NUM_HASH_BUCKETS; ++i) {
BrotliFree(m, words_by_hash[i]);
}
BrotliFree(m, words_by_hash);
BrotliFree(m, words_by_hash_size);
BrotliFree(m, words_by_hash_capacity);
BrotliTrieFree(m, &dedup);
return BROTLI_TRUE;
}
static void BuildDictionaryHashTable(uint16_t* hash_table_words,
uint8_t* hash_table_lengths, const BrotliDictionary* dict) {
int j, len;
/* The order of the loops is such that in case of collision, words with
shorter length are preferred, and in case of same length, words with
smaller index. There is only a single word per bucket. */
/* TODO(lode): consider adding optional user-supplied frequency_map to use
for preferred words instead, this can make the encoder better for
quality 9 and below without affecting the decoder */
memset(hash_table_words, 0, sizeof(kStaticDictionaryHashWords));
memset(hash_table_lengths, 0, sizeof(kStaticDictionaryHashLengths));
for (len = SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH;
len >= SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH; --len) {
const size_t num_words = dict->size_bits_by_length[len] ?
(1u << dict->size_bits_by_length[len]) : 0;
for (j = (int)num_words - 1; j >= 0; --j) {
size_t offset = dict->offsets_by_length[len] +
(size_t)len * (size_t)j;
const uint8_t* word = &dict->data[offset];
const uint32_t key = Hash(word, 14);
int idx = (int)(key << 1) + (len < 8 ? 1 : 0);
BROTLI_DCHECK(idx < (int)NUM_HASH_BUCKETS);
hash_table_words[idx] = (uint16_t)j;
hash_table_lengths[idx] = (uint8_t)len;
}
}
}
static BROTLI_BOOL GenerateWordsHeavy(MemoryManager* m,
const BrotliTransforms* transforms,
BrotliEncoderDictionary* dict) {
int i, j, l;
for (j = (int)transforms->num_transforms - 1; j >= 0 ; --j) {
for (l = 0; l < 32; l++) {
int num = (int)((1u << dict->words->size_bits_by_length[l]) & ~1u);
for (i = 0; i < num; i++) {
uint8_t transformed[kTransformedBufferSize];
size_t size;
TransformedDictionaryWord(
(uint32_t)i, l, j, transforms, dict, transformed, &size);
if (size < 4) continue;
if (!BrotliTrieAdd(m, (uint8_t)l, (uint32_t)(i + num * j),
transformed, size, &dict->trie)) {
return BROTLI_FALSE;
}
}
}
}
return BROTLI_TRUE;
}
/* Computes cutoffTransformsCount (in count) and cutoffTransforms (in data) for
the custom transforms, where possible within the limits of the
cutoffTransforms encoding. The fast encoder uses this to do fast lookup for
transforms that remove the N last characters (OmitLast). */
static void ComputeCutoffTransforms(
const BrotliTransforms* transforms,
uint32_t* count, uint64_t* data) {
int i;
/* The encoding in a 64-bit integer of transform N in the data is: (N << 2) +
((cutoffTransforms >> (N * 6)) & 0x3F), so for example the identity
transform code must be 0-63, for N=1 the transform code must be 4-67, ...,
for N=9 it must be 36-99.
TODO(lode): consider a simple flexible uint8_t[10] instead of the uint64_t
for the cutoff transforms, so that shared dictionaries can have the
OmitLast transforms anywhere without loss. */
*count = 0;
*data = 0;
for (i = 0; i < BROTLI_TRANSFORMS_MAX_CUT_OFF + 1; i++) {
int idx = transforms->cutOffTransforms[i];
if (idx == -1) break; /* Not found */
if (idx < (i << 2)) break; /* Too small for the encoding */
if (idx >= (i << 2) + 64) break; /* Too large for the encoding */
(*count)++;
*data |= (uint64_t)(((uint64_t)idx -
((uint64_t)i << 2u)) << ((uint64_t)i * 6u));
}
}
static BROTLI_BOOL ComputeDictionary(MemoryManager* m, int quality,
const BrotliTransforms* transforms,
BrotliEncoderDictionary* current) {
int default_words = current->words == BrotliGetDictionary();
int default_transforms = transforms == BrotliGetTransforms();
if (default_words && default_transforms) {
/* hashes are already set to Brotli defaults */
return BROTLI_TRUE;
}
current->hash_table_data_words_ = (uint16_t*)BrotliAllocate(
m, sizeof(kStaticDictionaryHashWords));
current->hash_table_data_lengths_ = (uint8_t*)BrotliAllocate(
m, sizeof(kStaticDictionaryHashLengths));
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
current->hash_table_words = current->hash_table_data_words_;
current->hash_table_lengths = current->hash_table_data_lengths_;
BuildDictionaryHashTable(current->hash_table_data_words_,
current->hash_table_data_lengths_, current->words);
ComputeCutoffTransforms(transforms,
&current->cutoffTransformsCount, &current->cutoffTransforms);
/* Only compute the data for slow encoder if the requested quality is high
enough to need it */
if (quality >= ZOPFLIFICATION_QUALITY) {
if (!BuildDictionaryLut(m, transforms, current)) return BROTLI_FALSE;
/* For the built-in Brotli transforms, there is a hard-coded function to
handle all transforms, but for custom transforms, we use the following
large hammer instead */
current->has_words_heavy = !default_transforms;
if (current->has_words_heavy) {
if (!GenerateWordsHeavy(m, transforms, current)) return BROTLI_FALSE;
}
}
return BROTLI_TRUE;
}
#endif /* BROTLI_EXPERIMENTAL */
void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) {
dict->magic = kSharedDictionaryMagic;
dict->compound.num_chunks = 0;
dict->compound.total_size = 0;
dict->compound.chunk_offsets[0] = 0;
dict->compound.num_prepared_instances_ = 0;
dict->contextual.context_based = 0;
dict->contextual.num_dictionaries = 1;
dict->contextual.instances_ = 0;
dict->contextual.num_instances_ = 1; /* The instance_ field */
dict->contextual.dict[0] = &dict->contextual.instance_;
InitEncoderDictionary(&dict->contextual.instance_);
dict->contextual.instance_.parent = &dict->contextual;
dict->max_quality = BROTLI_MAX_QUALITY;
}
#if defined(BROTLI_EXPERIMENTAL)
/* TODO(eustas): make sure that tooling will warn user if not all the cutoff
transforms are available (for low-quality encoder). */
static BROTLI_BOOL InitCustomSharedEncoderDictionary(
MemoryManager* m, const BrotliSharedDictionary* decoded_dict,
int quality, SharedEncoderDictionary* dict) {
ContextualEncoderDictionary* contextual;
CompoundDictionary* compound;
BrotliEncoderDictionary* instances;
int i;
BrotliInitSharedEncoderDictionary(dict);
contextual = &dict->contextual;
compound = &dict->compound;
for (i = 0; i < (int)decoded_dict->num_prefix; i++) {
PreparedDictionary* prepared = CreatePreparedDictionary(m,
decoded_dict->prefix[i], decoded_dict->prefix_size[i]);
AttachPreparedDictionary(compound, prepared);
/* remember for cleanup */
compound->prepared_instances_[
compound->num_prepared_instances_++] = prepared;
}
dict->max_quality = quality;
contextual->context_based = decoded_dict->context_based;
if (decoded_dict->context_based) {
memcpy(contextual->context_map, decoded_dict->context_map,
SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS);
}
contextual->num_dictionaries = decoded_dict->num_dictionaries;
contextual->num_instances_ = decoded_dict->num_dictionaries;
if (contextual->num_instances_ == 1) {
instances = &contextual->instance_;
} else {
contextual->instances_ = (BrotliEncoderDictionary*)
BrotliAllocate(m, sizeof(*contextual->instances_) *
contextual->num_instances_);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
instances = contextual->instances_;
}
for (i = 0; i < (int)contextual->num_instances_; i++) {
BrotliEncoderDictionary* current = &instances[i];
InitEncoderDictionary(current);
current->parent = &dict->contextual;
if (decoded_dict->words[i] == BrotliGetDictionary()) {
current->words = BrotliGetDictionary();
} else {
current->words_instance_ = (BrotliDictionary*)BrotliAllocate(
m, sizeof(BrotliDictionary));
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
*current->words_instance_ = *decoded_dict->words[i];
current->words = current->words_instance_;
}
current->num_transforms =
(uint32_t)decoded_dict->transforms[i]->num_transforms;
if (!ComputeDictionary(
m, quality, decoded_dict->transforms[i], current)) {
return BROTLI_FALSE;
}
contextual->dict[i] = current;
}
return BROTLI_TRUE; /* success */
}
BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
MemoryManager* m, const uint8_t* encoded_dict, size_t size,
int quality, SharedEncoderDictionary* dict) {
BROTLI_BOOL success = BROTLI_FALSE;
BrotliSharedDictionary* decoded_dict = BrotliSharedDictionaryCreateInstance(
m->alloc_func, m->free_func, m->opaque);
if (!decoded_dict) { /* OOM */
return BROTLI_FALSE;
}
success = BrotliSharedDictionaryAttach(
decoded_dict, BROTLI_SHARED_DICTIONARY_SERIALIZED, size, encoded_dict);
if (success) {
success = InitCustomSharedEncoderDictionary(m,
decoded_dict, quality, dict);
}
BrotliSharedDictionaryDestroyInstance(decoded_dict);
return success;
}
#endif /* BROTLI_EXPERIMENTAL */
void BrotliCleanupSharedEncoderDictionary(MemoryManager* m,
SharedEncoderDictionary* dict) {
size_t i;
for (i = 0; i < dict->compound.num_prepared_instances_; i++) {
DestroyPreparedDictionary(m,
(PreparedDictionary*)dict->compound.prepared_instances_[i]);
}
if (dict->contextual.num_instances_ == 1) {
BrotliDestroyEncoderDictionary(m, &dict->contextual.instance_);
} else if (dict->contextual.num_instances_ > 1) {
for (i = 0; i < dict->contextual.num_instances_; i++) {
BrotliDestroyEncoderDictionary(m, &dict->contextual.instances_[i]);
}
BrotliFree(m, dict->contextual.instances_);
}
}
ManagedDictionary* BrotliCreateManagedDictionary(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
ManagedDictionary* result = (ManagedDictionary*)BrotliBootstrapAlloc(
sizeof(ManagedDictionary), alloc_func, free_func, opaque);
if (result == NULL) return NULL;
result->magic = kManagedDictionaryMagic;
BrotliInitMemoryManager(
&result->memory_manager_, alloc_func, free_func, opaque);
result->dictionary = NULL;
return result;
}
void BrotliDestroyManagedDictionary(ManagedDictionary* dictionary) {
if (!dictionary) return;
BrotliBootstrapFree(dictionary, &dictionary->memory_manager_);
}
/* Escalate internal functions visibility; for testing purposes only. */
#if defined(BROTLI_TEST)
void BrotliInitEncoderDictionaryForTest(BrotliEncoderDictionary*);
void BrotliInitEncoderDictionaryForTest(BrotliEncoderDictionary* d) {
InitEncoderDictionary(d);
}
#endif
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,155 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#ifndef BROTLI_ENC_ENCODER_DICT_H_
#define BROTLI_ENC_ENCODER_DICT_H_
#include "../common/dictionary.h"
#include <brotli/shared_dictionary.h>
#include "../common/platform.h"
#include "compound_dictionary.h"
#include "memory.h"
#include "static_dict_lut.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/*
Dictionary hierarchy for Encoder:
-SharedEncoderDictionary
--CompoundDictionary
---PreparedDictionary [up to 15x]
= prefix dictionary with precomputed hashes
--ContextualEncoderDictionary
---BrotliEncoderDictionary [up to 64x]
= for each context, precomputed static dictionary with words + transforms
Dictionary hierarchy from common: similar, but without precomputed hashes
-BrotliSharedDictionary
--BrotliDictionary [up to 64x]
--BrotliTransforms [up to 64x]
--const uint8_t* prefix [up to 15x]: compound dictionaries
*/
typedef struct BrotliTrieNode {
uint8_t single; /* if 1, sub is a single node for c instead of 256 */
uint8_t c;
uint8_t len_; /* untransformed length */
uint32_t idx_; /* word index + num words * transform index */
uint32_t sub; /* index of sub node(s) in the pool */
} BrotliTrieNode;
typedef struct BrotliTrie {
BrotliTrieNode* pool;
size_t pool_capacity;
size_t pool_size;
BrotliTrieNode root;
} BrotliTrie;
#if defined(BROTLI_EXPERIMENTAL)
BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
const BrotliTrieNode* node, uint8_t c);
#endif /* BROTLI_EXPERIMENTAL */
/* Dictionary data (words and transforms) for 1 possible context */
typedef struct BrotliEncoderDictionary {
const BrotliDictionary* words;
uint32_t num_transforms;
/* cut off for fast encoder */
uint32_t cutoffTransformsCount;
uint64_t cutoffTransforms;
/* from dictionary_hash.h, for fast encoder */
const uint16_t* hash_table_words;
const uint8_t* hash_table_lengths;
/* from static_dict_lut.h, for slow encoder */
const uint16_t* buckets;
const DictWord* dict_words;
/* Heavy version, for use by slow encoder when there are custom transforms.
Contains every possible transformed dictionary word in a trie. It encodes
about as fast as the non-heavy encoder but consumes a lot of memory and
takes time to build. */
BrotliTrie trie;
BROTLI_BOOL has_words_heavy;
/* Reference to other dictionaries. */
const struct ContextualEncoderDictionary* parent;
/* Allocated memory, used only when not using the Brotli defaults */
uint16_t* hash_table_data_words_;
uint8_t* hash_table_data_lengths_;
size_t buckets_alloc_size_;
uint16_t* buckets_data_;
size_t dict_words_alloc_size_;
DictWord* dict_words_data_;
BrotliDictionary* words_instance_;
} BrotliEncoderDictionary;
/* Dictionary data for all 64 contexts */
typedef struct ContextualEncoderDictionary {
BROTLI_BOOL context_based;
uint8_t num_dictionaries;
uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
const BrotliEncoderDictionary* dict[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
/* If num_instances_ is 1, instance_ is used, else dynamic allocation with
instances_ is used. */
size_t num_instances_;
BrotliEncoderDictionary instance_;
BrotliEncoderDictionary* instances_;
} ContextualEncoderDictionary;
typedef struct SharedEncoderDictionary {
/* Magic value to distinguish this struct from PreparedDictionary for
certain external usages. */
uint32_t magic;
/* LZ77 prefix, compound dictionary */
CompoundDictionary compound;
/* Custom static dictionary (optionally context-based) */
ContextualEncoderDictionary contextual;
/* The maximum quality the dictionary was computed for */
int max_quality;
} SharedEncoderDictionary;
typedef struct ManagedDictionary {
uint32_t magic;
MemoryManager memory_manager_;
uint32_t* dictionary;
} ManagedDictionary;
/* Initializes to the brotli built-in dictionary */
BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary(
SharedEncoderDictionary* dict);
#if defined(BROTLI_EXPERIMENTAL)
/* Initializes to shared dictionary that will be parsed from
encoded_dict. Requires that you keep the encoded_dict buffer
around, parts of data will point to it. */
BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
MemoryManager* m, const uint8_t* encoded_dict, size_t size,
int quality, SharedEncoderDictionary* dict);
#endif /* BROTLI_EXPERIMENTAL */
BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary(
MemoryManager* m, SharedEncoderDictionary* dict);
BROTLI_INTERNAL ManagedDictionary* BrotliCreateManagedDictionary(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliDestroyManagedDictionary(
ManagedDictionary* dictionary);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_ENCODER_DICT_H_ */

View File

@@ -1,66 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Utilities for fast computation of logarithms. */
#ifndef BROTLI_ENC_FAST_LOG_H_
#define BROTLI_ENC_FAST_LOG_H_
#include <math.h>
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
#if defined(BROTLI_BSR32)
return BROTLI_BSR32((uint32_t)n);
#else
uint32_t result = 0;
while (n >>= 1) result++;
return result;
#endif
}
#define BROTLI_LOG2_TABLE_SIZE 256
/* A lookup table for small values of log2(int) to be used in entropy
computation. */
BROTLI_INTERNAL extern const BROTLI_MODEL("small")
double kBrotliLog2Table[BROTLI_LOG2_TABLE_SIZE];
/* Visual Studio 2012 and Android API levels < 18 do not have the log2()
* function defined, so we use log() and a multiplication instead. */
#if !defined(BROTLI_HAVE_LOG2)
#if ((defined(_MSC_VER) && _MSC_VER <= 1700) || \
(defined(__ANDROID_API__) && __ANDROID_API__ < 18))
#define BROTLI_HAVE_LOG2 0
#else
#define BROTLI_HAVE_LOG2 1
#endif
#endif
#define LOG_2_INV 1.4426950408889634
/* Faster logarithm for small integers, with the property of log2(0) == 0. */
static BROTLI_INLINE double FastLog2(size_t v) {
if (v < BROTLI_LOG2_TABLE_SIZE) {
return kBrotliLog2Table[v];
}
#if !(BROTLI_HAVE_LOG2)
return log((double)v) * LOG_2_INV;
#else
return log2((double)v);
#endif
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_FAST_LOG_H_ */

View File

@@ -1,735 +0,0 @@
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data. */
#ifndef BROTLI_ENC_HASH_H_
#define BROTLI_ENC_HASH_H_
#include "../common/constants.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "compound_dictionary.h"
#include "encoder_dict.h"
#include "fast_log.h"
#include "find_match_length.h"
#include "hash_base.h"
#include "matching_tag_mask.h"
#include "memory.h"
#include "params.h"
#include "quality.h"
#include "static_dict.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct {
/**
* Dynamically allocated areas; regular hasher uses one or two allocations;
* "composite" hasher uses up to 4 allocations.
*/
void* extra[4];
/**
* False before the first invocation of HasherSetup (where "extra" memory)
* is allocated.
*/
BROTLI_BOOL is_setup_;
size_t dict_num_lookups;
size_t dict_num_matches;
BrotliHasherParams params;
/**
* False if hasher needs to be "prepared" before use (before the first
* invocation of HasherSetup or after HasherReset). "preparation" is hasher
* data initialization (using input ringbuffer).
*/
BROTLI_BOOL is_prepared_;
} HasherCommon;
#define score_t size_t
static const uint32_t kCutoffTransformsCount = 10;
/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */
/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
static const uint64_t kCutoffTransforms =
BROTLI_MAKE_UINT64_T(0x071B520A, 0xDA2D3200);
typedef struct HasherSearchResult {
size_t len;
size_t distance;
score_t score;
int len_code_delta; /* == len_code - len */
} HasherSearchResult;
static BROTLI_INLINE void PrepareDistanceCache(
int* BROTLI_RESTRICT distance_cache, const int num_distances) {
if (num_distances > 4) {
int last_distance = distance_cache[0];
distance_cache[4] = last_distance - 1;
distance_cache[5] = last_distance + 1;
distance_cache[6] = last_distance - 2;
distance_cache[7] = last_distance + 2;
distance_cache[8] = last_distance - 3;
distance_cache[9] = last_distance + 3;
if (num_distances > 10) {
int next_last_distance = distance_cache[1];
distance_cache[10] = next_last_distance - 1;
distance_cache[11] = next_last_distance + 1;
distance_cache[12] = next_last_distance - 2;
distance_cache[13] = next_last_distance + 2;
distance_cache[14] = next_last_distance - 3;
distance_cache[15] = next_last_distance + 3;
}
}
}
#define BROTLI_LITERAL_BYTE_SCORE 135
#define BROTLI_DISTANCE_BIT_PENALTY 30
/* Score must be positive after applying maximal penalty. */
#define BROTLI_SCORE_BASE (BROTLI_DISTANCE_BIT_PENALTY * 8 * sizeof(size_t))
/* Usually, we always choose the longest backward reference. This function
allows for the exception of that rule.
If we choose a backward reference that is further away, it will
usually be coded with more bits. We approximate this by assuming
log2(distance). If the distance can be expressed in terms of the
last four distances, we use some heuristic constants to estimate
the bits cost. For the first up to four literals we use the bit
cost of the literals from the literal cost model, after that we
use the average bit cost of the cost model.
This function is used to sometimes discard a longer backward reference
when it is not much longer and the bit cost for encoding it is more
than the saved literals.
backward_reference_offset MUST be positive. */
static BROTLI_INLINE score_t BackwardReferenceScore(
size_t copy_length, size_t backward_reference_offset) {
return BROTLI_SCORE_BASE + BROTLI_LITERAL_BYTE_SCORE * (score_t)copy_length -
BROTLI_DISTANCE_BIT_PENALTY * Log2FloorNonZero(backward_reference_offset);
}
static BROTLI_INLINE score_t BackwardReferenceScoreUsingLastDistance(
size_t copy_length) {
return BROTLI_LITERAL_BYTE_SCORE * (score_t)copy_length +
BROTLI_SCORE_BASE + 15;
}
static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
size_t distance_short_code) {
return (score_t)39 + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE);
}
static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
const BrotliEncoderDictionary* dictionary, size_t len, size_t word_idx,
const uint8_t* data, size_t max_length, size_t max_backward,
size_t max_distance, HasherSearchResult* out) {
size_t offset;
size_t matchlen;
size_t backward;
score_t score;
offset = dictionary->words->offsets_by_length[len] + len * word_idx;
if (len > max_length) {
return BROTLI_FALSE;
}
matchlen =
FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
return BROTLI_FALSE;
}
{
size_t cut = len - matchlen;
size_t transform_id = (cut << 2) +
(size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
backward = max_backward + 1 + word_idx +
(transform_id << dictionary->words->size_bits_by_length[len]);
}
if (backward > max_distance) {
return BROTLI_FALSE;
}
score = BackwardReferenceScore(matchlen, backward);
if (score < out->score) {
return BROTLI_FALSE;
}
out->len = matchlen;
out->len_code_delta = (int)len - (int)matchlen;
out->distance = backward;
out->score = score;
return BROTLI_TRUE;
}
static BROTLI_INLINE void SearchInStaticDictionary(
const BrotliEncoderDictionary* dictionary,
HasherCommon* common, const uint8_t* data, size_t max_length,
size_t max_backward, size_t max_distance,
HasherSearchResult* out, BROTLI_BOOL shallow) {
size_t key;
size_t i;
if (common->dict_num_matches < (common->dict_num_lookups >> 7)) {
return;
}
key = Hash14(data) << 1;
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
common->dict_num_lookups++;
if (dictionary->hash_table_lengths[key] != 0) {
BROTLI_BOOL item_matches = TestStaticDictionaryItem(
dictionary, dictionary->hash_table_lengths[key],
dictionary->hash_table_words[key], data,
max_length, max_backward, max_distance, out);
if (item_matches) {
common->dict_num_matches++;
}
}
}
}
typedef struct BackwardMatch {
uint32_t distance;
uint32_t length_and_code;
} BackwardMatch;
static BROTLI_INLINE void InitBackwardMatch(BackwardMatch* self,
size_t dist, size_t len) {
self->distance = (uint32_t)dist;
self->length_and_code = (uint32_t)(len << 5);
}
static BROTLI_INLINE void InitDictionaryBackwardMatch(BackwardMatch* self,
size_t dist, size_t len, size_t len_code) {
self->distance = (uint32_t)dist;
self->length_and_code =
(uint32_t)((len << 5) | (len == len_code ? 0 : len_code));
}
static BROTLI_INLINE size_t BackwardMatchLength(const BackwardMatch* self) {
return self->length_and_code >> 5;
}
static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
size_t code = self->length_and_code & 31;
return code ? code : BackwardMatchLength(self);
}
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define HASHER() H10
#define BUCKET_BITS 17
#define MAX_TREE_SEARCH_DEPTH 64
#define MAX_TREE_COMP_LENGTH 128
#include "hash_to_binary_tree_inc.h" /* NOLINT(build/include) */
#undef MAX_TREE_SEARCH_DEPTH
#undef MAX_TREE_COMP_LENGTH
#undef BUCKET_BITS
#undef HASHER
/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
#define MAX_NUM_MATCHES_H10 128
/* For BUCKET_SWEEP_BITS == 0, enabling the dictionary lookup makes compression
a little faster (0.5% - 1%) and it compresses 0.15% better on small text
and HTML inputs. */
#define HASHER() H2
#define BUCKET_BITS 16
#define BUCKET_SWEEP_BITS 0
#define HASH_LEN 5
#define USE_DICTIONARY 1
#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef BUCKET_SWEEP_BITS
#undef USE_DICTIONARY
#undef HASHER
#define HASHER() H3
#define BUCKET_SWEEP_BITS 1
#define USE_DICTIONARY 0
#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef BUCKET_SWEEP_BITS
#undef BUCKET_BITS
#undef HASHER
#define HASHER() H4
#define BUCKET_BITS 17
#define BUCKET_SWEEP_BITS 2
#define USE_DICTIONARY 1
#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef HASH_LEN
#undef BUCKET_SWEEP_BITS
#undef BUCKET_BITS
#undef HASHER
#define HASHER() H5
#include "hash_longest_match_inc.h" /* NOLINT(build/include) */
#undef HASHER
#define HASHER() H6
#include "hash_longest_match64_inc.h" /* NOLINT(build/include) */
#undef HASHER
#if defined(BROTLI_MAX_SIMD_QUALITY)
#define HASHER() H58
#include "hash_longest_match_simd_inc.h" /* NOLINT(build/include) */
#undef HASHER
#define HASHER() H68
#include "hash_longest_match64_simd_inc.h" /* NOLINT(build/include) */
#undef HASHER
#endif
#define BUCKET_BITS 15
#define NUM_LAST_DISTANCES_TO_CHECK 4
#define NUM_BANKS 1
#define BANK_BITS 16
#define HASHER() H40
#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#define NUM_LAST_DISTANCES_TO_CHECK 10
#define HASHER() H41
#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#undef NUM_BANKS
#undef BANK_BITS
#define NUM_LAST_DISTANCES_TO_CHECK 16
#define NUM_BANKS 512
#define BANK_BITS 9
#define HASHER() H42
#include "hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
#undef HASHER
#undef NUM_LAST_DISTANCES_TO_CHECK
#undef NUM_BANKS
#undef BANK_BITS
#undef BUCKET_BITS
#define HASHER() H54
#define BUCKET_BITS 20
#define BUCKET_SWEEP_BITS 2
#define HASH_LEN 7
#define USE_DICTIONARY 0
#include "hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
#undef USE_DICTIONARY
#undef HASH_LEN
#undef BUCKET_SWEEP_BITS
#undef BUCKET_BITS
#undef HASHER
/* fast large window hashers */
#define HASHER() HROLLING_FAST
#define CHUNKLEN 32
#define JUMP 4
#define NUMBUCKETS 16777216
#define MASK ((NUMBUCKETS * 64) - 1)
#include "hash_rolling_inc.h" /* NOLINT(build/include) */
#undef JUMP
#undef HASHER
#define HASHER() HROLLING
#define JUMP 1
#include "hash_rolling_inc.h" /* NOLINT(build/include) */
#undef MASK
#undef NUMBUCKETS
#undef JUMP
#undef CHUNKLEN
#undef HASHER
#define HASHER() H35
#define HASHER_A H3
#define HASHER_B HROLLING_FAST
#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#define HASHER() H55
#define HASHER_A H54
#define HASHER_B HROLLING_FAST
#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#define HASHER() H65
#define HASHER_A H6
#define HASHER_B HROLLING
#include "hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#undef FN
#undef CAT
#undef EXPAND_CAT
#if defined(BROTLI_MAX_SIMD_QUALITY)
#define FOR_SIMPLE_HASHERS(H) \
H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54) H(58) H(68)
#else
#define FOR_SIMPLE_HASHERS(H) \
H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
#endif
#define FOR_COMPOSITE_HASHERS(H) H(35) H(55) H(65)
#define FOR_GENERIC_HASHERS(H) FOR_SIMPLE_HASHERS(H) FOR_COMPOSITE_HASHERS(H)
#define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
typedef struct {
HasherCommon common;
union {
#define MEMBER_(N) \
H ## N _H ## N;
FOR_ALL_HASHERS(MEMBER_)
#undef MEMBER_
} privat;
} Hasher;
/* MUST be invoked before any other method. */
static BROTLI_INLINE void HasherInit(Hasher* hasher) {
hasher->common.is_setup_ = BROTLI_FALSE;
hasher->common.extra[0] = NULL;
hasher->common.extra[1] = NULL;
hasher->common.extra[2] = NULL;
hasher->common.extra[3] = NULL;
}
static BROTLI_INLINE void DestroyHasher(MemoryManager* m, Hasher* hasher) {
if (hasher->common.extra[0] != NULL) BROTLI_FREE(m, hasher->common.extra[0]);
if (hasher->common.extra[1] != NULL) BROTLI_FREE(m, hasher->common.extra[1]);
if (hasher->common.extra[2] != NULL) BROTLI_FREE(m, hasher->common.extra[2]);
if (hasher->common.extra[3] != NULL) BROTLI_FREE(m, hasher->common.extra[3]);
}
static BROTLI_INLINE void HasherReset(Hasher* hasher) {
hasher->common.is_prepared_ = BROTLI_FALSE;
}
static BROTLI_INLINE void HasherSize(const BrotliEncoderParams* params,
BROTLI_BOOL one_shot, const size_t input_size, size_t* alloc_size) {
switch (params->hasher.type) {
#define SIZE_(N) \
case N: \
HashMemAllocInBytesH ## N(params, one_shot, input_size, alloc_size); \
break;
FOR_ALL_HASHERS(SIZE_)
#undef SIZE_
default:
break;
}
}
static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
BrotliEncoderParams* params, const uint8_t* data, size_t position,
size_t input_size, BROTLI_BOOL is_last) {
BROTLI_BOOL one_shot = (position == 0 && is_last);
if (!hasher->common.is_setup_) {
size_t alloc_size[4] = {0};
size_t i;
ChooseHasher(params, &params->hasher);
hasher->common.params = params->hasher;
hasher->common.dict_num_lookups = 0;
hasher->common.dict_num_matches = 0;
HasherSize(params, one_shot, input_size, alloc_size);
for (i = 0; i < 4; ++i) {
if (alloc_size[i] == 0) continue;
hasher->common.extra[i] = BROTLI_ALLOC(m, uint8_t, alloc_size[i]);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(hasher->common.extra[i])) return;
}
switch (hasher->common.params.type) {
#define INITIALIZE_(N) \
case N: \
InitializeH ## N(&hasher->common, \
&hasher->privat._H ## N, params); \
break;
FOR_ALL_HASHERS(INITIALIZE_);
#undef INITIALIZE_
default:
break;
}
HasherReset(hasher);
hasher->common.is_setup_ = BROTLI_TRUE;
}
if (!hasher->common.is_prepared_) {
switch (hasher->common.params.type) {
#define PREPARE_(N) \
case N: \
PrepareH ## N( \
&hasher->privat._H ## N, \
one_shot, input_size, data); \
break;
FOR_ALL_HASHERS(PREPARE_)
#undef PREPARE_
default: break;
}
hasher->common.is_prepared_ = BROTLI_TRUE;
}
}
static BROTLI_INLINE void InitOrStitchToPreviousBlock(
MemoryManager* m, Hasher* hasher, const uint8_t* data, size_t mask,
BrotliEncoderParams* params, size_t position, size_t input_size,
BROTLI_BOOL is_last) {
HasherSetup(m, hasher, params, data, position, input_size, is_last);
if (BROTLI_IS_OOM(m)) return;
switch (hasher->common.params.type) {
#define INIT_(N) \
case N: \
StitchToPreviousBlockH ## N( \
&hasher->privat._H ## N, \
input_size, position, data, mask); \
break;
FOR_ALL_HASHERS(INIT_)
#undef INIT_
default: break;
}
}
/* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget
to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */
static BROTLI_INLINE void FindCompoundDictionaryMatch(
const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t distance_offset,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
const uint32_t source_size = self->source_size;
const size_t boundary = distance_offset - source_size;
const uint32_t hash_bits = self->hash_bits;
const uint32_t bucket_bits = self->bucket_bits;
const uint32_t slot_bits = self->slot_bits;
const uint32_t hash_shift = 64u - bucket_bits;
const uint32_t slot_mask = (~((uint32_t)0U)) >> (32 - slot_bits);
const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]);
const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]);
const uint8_t* source = NULL;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
const uint64_t h =
(BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix_masked]) & hash_mask) *
kPreparedDictionaryHashMul64Long;
const uint32_t key = (uint32_t)(h >> hash_shift);
const uint32_t slot = key & slot_mask;
const uint32_t head = heads[key];
const uint32_t* BROTLI_RESTRICT chain = &items[slot_offsets[slot] + head];
uint32_t item = (head == 0xFFFF) ? 1 : 0;
const void* tail = (void*)&items[self->num_items];
if (self->magic == kPreparedDictionaryMagic) {
source = (const uint8_t*)tail;
} else {
/* kLeanPreparedDictionaryMagic */
source = (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
}
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
for (i = 0; i < 4; ++i) {
const size_t distance = (size_t)distance_cache[i];
size_t offset;
size_t limit;
size_t len;
if (distance <= boundary || distance > distance_offset) continue;
offset = distance_offset - distance;
limit = source_size - offset;
limit = limit > max_length ? max_length : limit;
len = FindMatchLengthWithLimit(&source[offset], &data[cur_ix_masked],
limit);
if (len >= 2) {
score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
if (best_score < score) {
best_score = score;
if (len > best_len) best_len = len;
out->len = len;
out->len_code_delta = 0;
out->distance = distance;
out->score = best_score;
}
}
}
}
/* we require matches of len >4, so increase best_len to 3, so we can compare
* 4 bytes all the time. */
if (best_len < 3) {
best_len = 3;
}
while (item == 0) {
size_t offset;
size_t distance;
size_t limit;
item = *chain;
chain++;
offset = item & 0x7FFFFFFF;
item &= 0x80000000;
distance = distance_offset - offset;
limit = source_size - offset;
limit = (limit > max_length) ? max_length : limit;
if (distance > max_distance) continue;
if (cur_ix_masked + best_len > ring_buffer_mask || best_len >= limit ||
/* compare 4 bytes ending at best_len + 1 */
BrotliUnalignedRead32(&data[cur_ix_masked + best_len - 3]) !=
BrotliUnalignedRead32(&source[offset + best_len - 3])) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&source[offset],
&data[cur_ix_masked],
limit);
if (len >= 4) {
score_t score = BackwardReferenceScore(len, distance);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->len_code_delta = 0;
out->distance = distance;
out->score = best_score;
}
}
}
}
}
/* NB: when seamless dictionary-ring-buffer copies are implemented, don't forget
to add proper guards for non-zero-BROTLI_PARAM_STREAM_OFFSET. */
static BROTLI_INLINE size_t FindAllCompoundDictionaryMatches(
const PreparedDictionary* self, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const size_t cur_ix, const size_t min_length,
const size_t max_length, const size_t distance_offset,
const size_t max_distance, BackwardMatch* matches, size_t match_limit) {
const uint32_t source_size = self->source_size;
const uint32_t hash_bits = self->hash_bits;
const uint32_t bucket_bits = self->bucket_bits;
const uint32_t slot_bits = self->slot_bits;
const uint32_t hash_shift = 64u - bucket_bits;
const uint32_t slot_mask = (~((uint32_t)0U)) >> (32 - slot_bits);
const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]);
const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]);
const uint8_t* source = NULL;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t best_len = min_length;
const uint64_t h =
(BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix_masked]) & hash_mask) *
kPreparedDictionaryHashMul64Long;
const uint32_t key = (uint32_t)(h >> hash_shift);
const uint32_t slot = key & slot_mask;
const uint32_t head = heads[key];
const uint32_t* BROTLI_RESTRICT chain = &items[slot_offsets[slot] + head];
uint32_t item = (head == 0xFFFF) ? 1 : 0;
size_t found = 0;
const void* tail = (void*)&items[self->num_items];
if (self->magic == kPreparedDictionaryMagic) {
source = (const uint8_t*)tail;
} else {
/* kLeanPreparedDictionaryMagic */
source = (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
}
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
while (item == 0) {
size_t offset;
size_t distance;
size_t limit;
size_t len;
item = *chain;
chain++;
offset = item & 0x7FFFFFFF;
item &= 0x80000000;
distance = distance_offset - offset;
limit = source_size - offset;
limit = (limit > max_length) ? max_length : limit;
if (distance > max_distance) continue;
if (cur_ix_masked + best_len > ring_buffer_mask ||
best_len >= limit ||
data[cur_ix_masked + best_len] != source[offset + best_len]) {
continue;
}
len = FindMatchLengthWithLimit(
&source[offset], &data[cur_ix_masked], limit);
if (len > best_len) {
best_len = len;
InitBackwardMatch(matches++, distance, len);
found++;
if (found == match_limit) break;
}
}
return found;
}
static BROTLI_INLINE void LookupCompoundDictionaryMatch(
const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length,
const size_t max_ring_buffer_distance, const size_t max_distance,
HasherSearchResult* sr) {
size_t base_offset = max_ring_buffer_distance + 1 + addon->total_size - 1;
size_t d;
for (d = 0; d < addon->num_chunks; ++d) {
/* Only one prepared dictionary type is currently supported. */
FindCompoundDictionaryMatch(
(const PreparedDictionary*)addon->chunks[d], data, ring_buffer_mask,
distance_cache, cur_ix, max_length,
base_offset - addon->chunk_offsets[d], max_distance, sr);
}
}
static BROTLI_INLINE size_t LookupAllCompoundDictionaryMatches(
const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const size_t cur_ix, size_t min_length,
const size_t max_length, const size_t max_ring_buffer_distance,
const size_t max_distance, BackwardMatch* matches,
size_t match_limit) {
size_t base_offset = max_ring_buffer_distance + 1 + addon->total_size - 1;
size_t d;
size_t total_found = 0;
for (d = 0; d < addon->num_chunks; ++d) {
/* Only one prepared dictionary type is currently supported. */
total_found += FindAllCompoundDictionaryMatches(
(const PreparedDictionary*)addon->chunks[d], data, ring_buffer_mask,
cur_ix, min_length, max_length, base_offset - addon->chunk_offsets[d],
max_distance, matches + total_found, match_limit - total_found);
if (total_found == match_limit) break;
if (total_found > 0) {
min_length = BackwardMatchLength(&matches[total_found - 1]);
}
}
return total_found;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_HASH_H_ */

View File

@@ -1,38 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Basic common hash functions / constants. */
#ifndef THIRD_PARTY_BROTLI_ENC_HASH_BASE_H_
#define THIRD_PARTY_BROTLI_ENC_HASH_BASE_H_
#include "../common/platform.h"
/* kHashMul32 multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit.
* No long streaks of ones or zeros.
* There is no effort to ensure that it is a prime, the oddity is enough
for this use.
* The number has been tuned heuristically against compression benchmarks. */
static const uint32_t kHashMul32 = 0x1E35A7BD;
static const uint64_t kHashMul64 =
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - 14);
}
static BROTLI_INLINE uint32_t Hash15(const uint8_t* data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - 15);
}
#endif // THIRD_PARTY_BROTLI_ENC_HASH_BASE_H_

View File

@@ -1,140 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, HASHER_A, HASHER_B */
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
and HASHER_B. */
#define HashComposite HASHER()
#define FN_A(X) EXPAND_CAT(X, HASHER_A)
#define FN_B(X) EXPAND_CAT(X, HASHER_B)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
size_t a = FN_A(HashTypeLength)();
size_t b = FN_B(HashTypeLength)();
return a > b ? a : b;
}
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
size_t a = FN_A(StoreLookahead)();
size_t b = FN_B(StoreLookahead)();
return a > b ? a : b;
}
typedef struct HashComposite {
HASHER_A ha;
HASHER_B hb;
HasherCommon ha_common;
HasherCommon hb_common;
/* Shortcuts. */
HasherCommon* common;
BROTLI_BOOL fresh;
const BrotliEncoderParams* params;
} HashComposite;
static void FN(Initialize)(HasherCommon* common,
HashComposite* BROTLI_RESTRICT self, const BrotliEncoderParams* params) {
self->common = common;
self->ha_common = *self->common;
self->hb_common = *self->common;
self->fresh = BROTLI_TRUE;
self->params = params;
/* TODO(lode): Initialize of the hashers is deferred to Prepare (and params
remembered here) because we don't get the one_shot and input_size params
here that are needed to know the memory size of them. Instead provide
those params to all hashers FN(Initialize) */
}
static void FN(Prepare)(
HashComposite* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
if (self->fresh) {
self->fresh = BROTLI_FALSE;
self->ha_common.extra[0] = self->common->extra[0];
self->ha_common.extra[1] = self->common->extra[1];
self->ha_common.extra[2] = NULL;
self->ha_common.extra[3] = NULL;
self->hb_common.extra[0] = self->common->extra[2];
self->hb_common.extra[1] = self->common->extra[3];
self->hb_common.extra[2] = NULL;
self->hb_common.extra[3] = NULL;
FN_A(Initialize)(&self->ha_common, &self->ha, self->params);
FN_B(Initialize)(&self->hb_common, &self->hb, self->params);
}
FN_A(Prepare)(&self->ha, one_shot, input_size, data);
FN_B(Prepare)(&self->hb, one_shot, input_size, data);
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t alloc_size_a[4] = {0};
size_t alloc_size_b[4] = {0};
FN_A(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_a);
FN_B(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_b);
/* Should never happen. */
if (alloc_size_a[2] != 0 || alloc_size_a[3] != 0) exit(EXIT_FAILURE);
if (alloc_size_b[2] != 0 || alloc_size_b[3] != 0) exit(EXIT_FAILURE);
alloc_size[0] = alloc_size_a[0];
alloc_size[1] = alloc_size_a[1];
alloc_size[2] = alloc_size_b[0];
alloc_size[3] = alloc_size_b[1];
}
static BROTLI_INLINE void FN(Store)(HashComposite* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
FN_A(Store)(&self->ha, data, mask, ix);
FN_B(Store)(&self->hb, data, mask, ix);
}
static BROTLI_INLINE void FN(StoreRange)(
HashComposite* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix_start,
const size_t ix_end) {
FN_A(StoreRange)(&self->ha, data, mask, ix_start, ix_end);
FN_B(StoreRange)(&self->hb, data, mask, ix_start, ix_end);
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashComposite* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ring_buffer_mask) {
FN_A(StitchToPreviousBlock)(&self->ha, num_bytes, position,
ringbuffer, ring_buffer_mask);
FN_B(StitchToPreviousBlock)(&self->hb, num_bytes, position,
ringbuffer, ring_buffer_mask);
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashComposite* BROTLI_RESTRICT self, int* BROTLI_RESTRICT distance_cache) {
FN_A(PrepareDistanceCache)(&self->ha, distance_cache);
FN_B(PrepareDistanceCache)(&self->hb, distance_cache);
}
static BROTLI_INLINE void FN(FindLongestMatch)(
HashComposite* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
FN_A(FindLongestMatch)(&self->ha, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, dictionary_distance,
max_distance, out);
FN_B(FindLongestMatch)(&self->hb, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, dictionary_distance,
max_distance, out);
}
#undef HashComposite

View File

@@ -1,279 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
#define HashLongestMatch HASHER()
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data,
uint64_t hash_mul) {
const uint64_t h = BROTLI_UNALIGNED_LOAD64LE(data) * hash_mul;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (size_t)(h >> (64 - 15));
}
typedef struct HashLongestMatch {
/* Number of hash buckets. */
size_t bucket_size_;
/* Only block_size_ newest backward references are kept,
and the older are forgotten. */
size_t block_size_;
/* Hash multiplier tuned to match length. */
uint64_t hash_mul_;
/* Mask for accessing entries in a block (in a ring-buffer manner). */
uint32_t block_mask_;
int block_bits_;
int num_last_distances_to_check_;
/* Shortcuts. */
HasherCommon* common_;
/* --- Dynamic size members --- */
/* Number of entries in a particular bucket. */
uint16_t* num_; /* uint16_t[bucket_size]; */
/* Buckets containing block_size_ of backward references. */
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common_ = common;
BROTLI_UNUSED(params);
self->hash_mul_ = kHashMul64 << (64 - 5 * 8);
BROTLI_DCHECK(common->params.bucket_bits == 15);
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_bits_ = common->params.block_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
self->num_ = (uint16_t*)common->extra[0];
self->buckets_ = (uint32_t*)common->extra[1];
}
static void FN(Prepare)(
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint16_t* BROTLI_RESTRICT num = self->num_;
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
const size_t key = FN(HashBytes)(&data[i], self->hash_mul_);
num[key] = 0;
}
} else {
memset(num, 0, self->bucket_size_ * sizeof(num[0]));
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
static BROTLI_INLINE void FN(Store)(
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t key = FN(HashBytes)(&data[ix & mask], self->hash_mul_);
const size_t minor_ix = num[key] & self->block_mask_;
const size_t offset = minor_ix + (key << self->block_bits_);
++num[key];
buckets[offset] = (uint32_t)ix;
}
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashLongestMatch* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
}
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashLongestMatch* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
}
/* Find a longest backward match of &data[cur_ix] up to the length of
max_length and stores the position cur_ix in the hash table.
REQUIRES: FN(PrepareDistanceCache) must be invoked for current distance cache
values; if this method is invoked repeatedly with the same distance
cache values, it is enough to invoke FN(PrepareDistanceCache) once.
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
/* Precalculate the hash key and prefetch the bucket. */
const size_t key = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
PREFETCH_L1(bucket);
if (self->block_bits_ > 4) PREFETCH_L1(bucket + 16);
out->len = 0;
out->len_code_delta = 0;
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
/* Try last distance first. */
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
const size_t backward = (size_t)distance_cache[i];
size_t prev_ix = (size_t)(cur_ix - backward);
if (prev_ix >= cur_ix) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
continue;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 3 || (len == 2 && i < 2)) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
}
/* we require matches of len >4, so increase best_len to 3, so we can compare
* 4 bytes all the time. */
if (best_len < 3) {
best_len = 3;
}
{
const size_t down =
(num[key] > self->block_size_) ?
(num[key] - self->block_size_) : 0u;
const uint32_t first4 = BrotliUnalignedRead32(data + cur_ix_masked);
const size_t max_length_m4 = max_length - 4;
i = num[key];
for (; i > down;) {
size_t prev_ix = bucket[--i & self->block_mask_];
uint32_t current4;
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
/* compare 4 bytes ending at best_len + 1 */
BrotliUnalignedRead32(&data[cur_ix_masked + best_len - 3]) !=
BrotliUnalignedRead32(&data[prev_ix + best_len - 3])) {
continue;
}
current4 = BrotliUnalignedRead32(data + prev_ix);
if (first4 != current4) continue;
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix + 4],
&data[cur_ix_masked + 4],
max_length_m4) + 4;
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary,
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
max_distance, out, BROTLI_FALSE);
}
}
#undef HashLongestMatch

View File

@@ -1,304 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
#define HashLongestMatch HASHER()
#define TAG_HASH_BITS 8
#define TAG_HASH_MASK ((1 << TAG_HASH_BITS) - 1)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data,
uint64_t hash_mul) {
const uint64_t h = BROTLI_UNALIGNED_LOAD64LE(data) * hash_mul;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (size_t)(h >> (64 - 15 - TAG_HASH_BITS));
}
typedef struct HashLongestMatch {
/* Number of hash buckets. */
size_t bucket_size_;
/* Only block_size_ newest backward references are kept,
and the older are forgotten. */
size_t block_size_;
/* Hash multiplier tuned to match length. */
uint64_t hash_mul_;
/* Mask for accessing entries in a block (in a ring-buffer manner). */
uint32_t block_mask_;
int block_bits_;
int num_last_distances_to_check_;
/* Shortcuts. */
HasherCommon* common_;
/* --- Dynamic size members --- */
/* Number of entries in a particular bucket. */
uint16_t* num_; /* uint16_t[bucket_size]; */
uint8_t* tags_;
/* Buckets containing block_size_ of backward references. */
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common_ = common;
BROTLI_UNUSED(params);
self->hash_mul_ = kHashMul64 << (64 - 5 * 8);
BROTLI_DCHECK(common->params.bucket_bits == 15);
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_bits_ = common->params.block_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
self->num_ = (uint16_t*)common->extra[0];
self->tags_ = (uint8_t*)common->extra[1];
self->buckets_ = (uint32_t*)common->extra[2];
}
static void FN(Prepare)(
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint16_t* BROTLI_RESTRICT num = self->num_;
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
const size_t hash = FN(HashBytes)(&data[i], self->hash_mul_);
const size_t key = hash >> TAG_HASH_BITS;
num[key] = 65535;
}
} else {
/* Set all the bytes of num to 255, which makes each uint16_t 65535. */
memset(num, 255, self->bucket_size_ * sizeof(num[0]));
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint8_t) * bucket_size * block_size;
alloc_size[2] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
static BROTLI_INLINE void FN(Store)(
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint8_t* BROTLI_RESTRICT tags = self->tags_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t hash = FN(HashBytes)(&data[ix & mask], self->hash_mul_);
const size_t key = hash >> TAG_HASH_BITS;
const uint8_t tag = hash & TAG_HASH_MASK;
const size_t minor_ix = num[key] & self->block_mask_;
const size_t offset = minor_ix + (key << self->block_bits_);
--num[key];
buckets[offset] = (uint32_t)ix;
tags[offset] = tag;
}
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashLongestMatch* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
}
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashLongestMatch* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
}
/* Find a longest backward match of &data[cur_ix] up to the length of
max_length and stores the position cur_ix in the hash table.
REQUIRES: FN(PrepareDistanceCache) must be invoked for current distance cache
values; if this method is invoked repeatedly with the same distance
cache values, it is enough to invoke FN(PrepareDistanceCache) once.
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
uint8_t* BROTLI_RESTRICT tags = self->tags_;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
/* Precalculate the hash key and prefetch the bucket. */
const size_t hash = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
const size_t key = hash >> TAG_HASH_BITS;
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
uint8_t* BROTLI_RESTRICT tag_bucket = &tags[key << self->block_bits_];
PREFETCH_L1(bucket);
PREFETCH_L1(tag_bucket);
if (self->block_bits_ > 4) PREFETCH_L1(bucket + 16);
out->len = 0;
out->len_code_delta = 0;
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
/* Try last distance first. */
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
const size_t backward = (size_t)distance_cache[i];
size_t prev_ix = (size_t)(cur_ix - backward);
if (prev_ix >= cur_ix) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
continue;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 3 || (len == 2 && i < 2)) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
}
/* we require matches of len >4, so increase best_len to 3, so we can compare
* 4 bytes all the time. */
if (best_len < 3) {
best_len = 3;
}
{
const uint8_t tag = hash & TAG_HASH_MASK;
const uint32_t first4 = BrotliUnalignedRead32(data + cur_ix_masked);
const size_t max_length_m4 = max_length - 4;
const size_t head = (num[key] + 1) & self->block_mask_;
uint64_t matches =
GetMatchingTagMask(self->block_size_ / 16, tag, tag_bucket, head);
/* Mask off any matches from uninitialized tags. */
uint16_t n = 65535 - num[key];
uint64_t block_has_unused_slots = self->block_size_ > n;
uint64_t mask = (block_has_unused_slots << (n & (64 - 1))) - 1;
matches &= mask;
for (; matches > 0; matches &= (matches - 1)) {
const size_t rb_index =
(head + (size_t)BROTLI_TZCNT64(matches)) & self->block_mask_;
size_t prev_ix = bucket[rb_index];
uint32_t current4;
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
/* compare 4 bytes ending at best_len + 1 */
BrotliUnalignedRead32(&data[cur_ix_masked + best_len - 3]) !=
BrotliUnalignedRead32(&data[prev_ix + best_len - 3])) {
continue;
}
current4 = BrotliUnalignedRead32(data + prev_ix);
if (first4 != current4) continue;
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix + 4],
&data[cur_ix_masked + 4],
max_length_m4) + 4;
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
tag_bucket[num[key] & self->block_mask_] = tag;
--num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary,
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
max_distance, out, BROTLI_FALSE);
}
}
#undef HashLongestMatch

View File

@@ -1,277 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
#define HashLongestMatch HASHER()
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static uint32_t FN(HashBytes)(
const uint8_t* BROTLI_RESTRICT data, const int shift) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
}
typedef struct HashLongestMatch {
/* Number of hash buckets. */
size_t bucket_size_;
/* Only block_size_ newest backward references are kept,
and the older are forgotten. */
size_t block_size_;
/* Left-shift for computing hash bucket index from hash value. */
int hash_shift_;
/* Mask for accessing entries in a block (in a ring-buffer manner). */
uint32_t block_mask_;
int block_bits_;
int num_last_distances_to_check_;
/* Shortcuts. */
HasherCommon* common_;
/* --- Dynamic size members --- */
/* Number of entries in a particular bucket. */
uint16_t* num_; /* uint16_t[bucket_size]; */
/* Buckets containing block_size_ of backward references. */
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common_ = common;
BROTLI_UNUSED(params);
self->hash_shift_ = 32 - common->params.bucket_bits;
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_ = (uint16_t*)common->extra[0];
self->buckets_ = (uint32_t*)common->extra[1];
self->block_bits_ = common->params.block_bits;
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
}
static void FN(Prepare)(
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint16_t* BROTLI_RESTRICT num = self->num_;
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
const uint32_t key = FN(HashBytes)(&data[i], self->hash_shift_);
num[key] = 0;
}
} else {
memset(num, 0, self->bucket_size_ * sizeof(num[0]));
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
static BROTLI_INLINE void FN(Store)(
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const uint32_t key = FN(HashBytes)(&data[ix & mask], self->hash_shift_);
const size_t minor_ix = num[key] & self->block_mask_;
const size_t offset = minor_ix + (key << self->block_bits_);
++num[key];
buckets[offset] = (uint32_t)ix;
}
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashLongestMatch* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
}
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashLongestMatch* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
}
/* Find a longest backward match of &data[cur_ix] up to the length of
max_length and stores the position cur_ix in the hash table.
REQUIRES: FN(PrepareDistanceCache) must be invoked for current distance cache
values; if this method is invoked repeatedly with the same distance
cache values, it is enough to invoke FN(PrepareDistanceCache) once.
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
/* Precalculate the hash key and prefetch the bucket. */
const uint32_t key =
FN(HashBytes)(&data[cur_ix_masked], self->hash_shift_);
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
PREFETCH_L1(bucket);
if (self->block_bits_ > 4) PREFETCH_L1(bucket + 16);
out->len = 0;
out->len_code_delta = 0;
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
/* Try last distance first. */
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
const size_t backward = (size_t)distance_cache[i];
size_t prev_ix = (size_t)(cur_ix - backward);
if (prev_ix >= cur_ix) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
continue;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 3 || (len == 2 && i < 2)) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
}
/* we require matches of len >4, so increase best_len to 3, so we can compare
* 4 bytes all the time. */
if (best_len < 3) {
best_len = 3;
}
{
const size_t down =
(num[key] > self->block_size_) ? (num[key] - self->block_size_) : 0u;
for (i = num[key]; i > down;) {
size_t prev_ix = bucket[--i & self->block_mask_];
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
/* compare 4 bytes ending at best_len + 1 */
BrotliUnalignedRead32(&data[cur_ix_masked + best_len - 3]) !=
BrotliUnalignedRead32(&data[prev_ix + best_len - 3])) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
/* Comparing for >= 3 does not change the semantics, but just saves
for a few unnecessary binary logarithms in backward reference
score, since we are not interested in such short matches. */
score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary,
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
max_distance, out, BROTLI_FALSE);
}
}
#undef HashLongestMatch

View File

@@ -1,270 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, BUCKET_BITS, BUCKET_SWEEP_BITS, HASH_LEN,
USE_DICTIONARY
*/
#define HashLongestMatchQuickly HASHER()
#define BUCKET_SIZE (1 << BUCKET_BITS)
#define BUCKET_MASK (BUCKET_SIZE - 1)
#define BUCKET_SWEEP (1 << BUCKET_SWEEP_BITS)
#define BUCKET_SWEEP_MASK ((BUCKET_SWEEP - 1) << 3)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
/* HashBytes is the function that chooses the bucket to place
the address in. The HashLongestMatch and HashLongestMatchQuickly
classes have separate, different implementations of hashing. */
static uint32_t FN(HashBytes)(const uint8_t* data) {
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64LE(data) << (64 - 8 * HASH_LEN)) *
kHashMul64);
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> (64 - BUCKET_BITS));
}
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
This is a hash map of fixed size (BUCKET_SIZE). */
typedef struct HashLongestMatchQuickly {
/* Shortcuts. */
HasherCommon* common;
/* --- Dynamic size members --- */
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
} HashLongestMatchQuickly;
static void FN(Initialize)(
HasherCommon* common, HashLongestMatchQuickly* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common = common;
BROTLI_UNUSED(params);
self->buckets_ = (uint32_t*)common->extra[0];
}
static void FN(Prepare)(
HashLongestMatchQuickly* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = BUCKET_SIZE >> 5;
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
const uint32_t key = FN(HashBytes)(&data[i]);
if (BUCKET_SWEEP == 1) {
buckets[key] = 0;
} else {
uint32_t j;
for (j = 0; j < BUCKET_SWEEP; ++j) {
buckets[(key + (j << 3)) & BUCKET_MASK] = 0;
}
}
}
} else {
/* It is not strictly necessary to fill this buffer here, but
not filling will make the results of the compression stochastic
(but correct). This is because random data would cause the
system to find accidentally good backward references here and there. */
memset(buckets, 0, sizeof(uint32_t) * BUCKET_SIZE);
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
}
/* Look at 5 bytes at &data[ix & mask].
Compute a hash from these, and store the value somewhere within
[ix .. ix+3]. */
static BROTLI_INLINE void FN(Store)(
HashLongestMatchQuickly* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
const uint32_t key = FN(HashBytes)(&data[ix & mask]);
if (BUCKET_SWEEP == 1) {
self->buckets_[key] = (uint32_t)ix;
} else {
/* Wiggle the value with the bucket sweep range. */
const uint32_t off = ix & BUCKET_SWEEP_MASK;
self->buckets_[(key + off) & BUCKET_MASK] = (uint32_t)ix;
}
}
static BROTLI_INLINE void FN(StoreRange)(
HashLongestMatchQuickly* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashLongestMatchQuickly* BROTLI_RESTRICT self,
size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
}
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashLongestMatchQuickly* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
BROTLI_UNUSED(self);
BROTLI_UNUSED(distance_cache);
}
/* Find a longest backward match of &data[cur_ix & ring_buffer_mask]
up to the length of max_length and stores the position cur_ix in the
hash table.
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HashLongestMatchQuickly* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t best_len_in = out->len;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* TODO: compare 4 bytes at once (and set the minimum best len to 4) */
int compare_char = data[cur_ix_masked + best_len_in];
size_t key = FN(HashBytes)(&data[cur_ix_masked]);
size_t key_out;
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = best_len_in;
size_t cached_backward = (size_t)distance_cache[0];
size_t prev_ix = cur_ix - cached_backward;
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
out->len_code_delta = 0;
if (prev_ix < cur_ix) {
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char == data[prev_ix + best_len]) {
const size_t len = FindMatchLengthWithLimit(
&data[prev_ix], &data[cur_ix_masked], max_length);
if (len >= 4) {
const score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
out->len = len;
out->distance = cached_backward;
out->score = score;
if (BUCKET_SWEEP == 1) {
buckets[key] = (uint32_t)cur_ix;
return;
} else {
best_len = len;
best_score = score;
compare_char = data[cur_ix_masked + len];
}
}
}
}
}
if (BUCKET_SWEEP == 1) {
size_t backward;
size_t len;
/* Only one to look for, don't bother to prepare for a loop. */
prev_ix = buckets[key];
buckets[key] = (uint32_t)cur_ix;
backward = cur_ix - prev_ix;
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char != data[prev_ix + best_len_in]) {
return;
}
if (BROTLI_PREDICT_FALSE(backward == 0 || backward > max_backward)) {
return;
}
len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
out->len = len;
out->distance = backward;
out->score = score;
return;
}
}
} else {
size_t keys[BUCKET_SWEEP];
size_t i;
for (i = 0; i < BUCKET_SWEEP; ++i) {
keys[i] = (key + (i << 3)) & BUCKET_MASK;
}
key_out = keys[(cur_ix & BUCKET_SWEEP_MASK) >> 3];
for (i = 0; i < BUCKET_SWEEP; ++i) {
size_t len;
size_t backward;
prev_ix = buckets[keys[i]];
backward = cur_ix - prev_ix;
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char != data[prev_ix + best_len]) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward == 0 || backward > max_backward)) {
continue;
}
len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
best_len = len;
out->len = len;
compare_char = data[cur_ix_masked + len];
best_score = score;
out->score = score;
out->distance = backward;
}
}
}
}
if (USE_DICTIONARY && min_score == out->score) {
SearchInStaticDictionary(dictionary,
self->common, &data[cur_ix_masked], max_length, dictionary_distance,
max_distance, out, BROTLI_TRUE);
}
if (BUCKET_SWEEP != 1) {
buckets[key_out] = (uint32_t)cur_ix;
}
}
#undef BUCKET_SWEEP_MASK
#undef BUCKET_SWEEP
#undef BUCKET_MASK
#undef BUCKET_SIZE
#undef HashLongestMatchQuickly

View File

@@ -1,278 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
#define HashLongestMatch HASHER()
#define TAG_HASH_BITS 8
#define TAG_HASH_MASK ((1 << TAG_HASH_BITS) - 1)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static uint32_t FN(HashBytes)(
const uint8_t* BROTLI_RESTRICT data, const int shift) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
}
typedef struct HashLongestMatch {
/* Number of hash buckets. */
size_t bucket_size_;
/* Only block_size_ newest backward references are kept,
and the older are forgotten. */
size_t block_size_;
/* Left-shift for computing hash bucket index from hash value. */
int hash_shift_;
/* Mask for accessing entries in a block (in a ring-buffer manner). */
uint32_t block_mask_;
int block_bits_;
int num_last_distances_to_check_;
/* Shortcuts. */
HasherCommon* common_;
/* --- Dynamic size members --- */
/* Number of entries in a particular bucket. */
uint16_t* num_; /* uint16_t[bucket_size]; */
uint8_t* tags_;
/* Buckets containing block_size_ of backward references. */
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common_ = common;
BROTLI_UNUSED(params);
self->hash_shift_ = 32 - common->params.bucket_bits - TAG_HASH_BITS;
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_ = (uint16_t*)common->extra[0];
self->tags_ = (uint8_t*)common->extra[1];
self->buckets_ = (uint32_t*)common->extra[2];
self->block_bits_ = common->params.block_bits;
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
}
static void FN(Prepare)(
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint16_t* BROTLI_RESTRICT num = self->num_;
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
size_t i;
for (i = 0; i < input_size; ++i) {
const uint32_t hash = FN(HashBytes)(&data[i], self->hash_shift_);
const uint32_t key = hash >> TAG_HASH_BITS;
num[key] = 65535;
}
} else {
/* Set all the bytes of num to 255, which makes each uint16_t 65535. */
memset(num, 255, self->bucket_size_ * sizeof(num[0]));
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint8_t) * bucket_size * block_size;
alloc_size[2] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
static BROTLI_INLINE void FN(Store)(
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint8_t* BROTLI_RESTRICT tags = self->tags_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
const size_t hash = FN(HashBytes)(&data[ix & mask], self->hash_shift_);
const size_t key = hash >> TAG_HASH_BITS;
const uint8_t tag = hash & TAG_HASH_MASK;
const size_t minor_ix = num[key] & self->block_mask_;
const size_t offset = minor_ix + (key << self->block_bits_);
--num[key];
buckets[offset] = (uint32_t)ix;
tags[offset] = tag;
}
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashLongestMatch* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
}
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashLongestMatch* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
}
/* Find a longest backward match of &data[cur_ix] up to the length of
max_length and stores the position cur_ix in the hash table.
REQUIRES: FN(PrepareDistanceCache) must be invoked for current distance cache
values; if this method is invoked repeatedly with the same distance
cache values, it is enough to invoke FN(PrepareDistanceCache) once.
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint16_t* BROTLI_RESTRICT num = self->num_;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
uint8_t* BROTLI_RESTRICT tags = self->tags_;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
/* Precalculate the hash key and prefetch the bucket. */
const uint32_t hash =
FN(HashBytes)(&data[cur_ix_masked], self->hash_shift_);
const uint32_t key = hash >> TAG_HASH_BITS;
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
uint8_t* BROTLI_RESTRICT tag_bucket = &tags[key << self->block_bits_];
PREFETCH_L1(bucket);
PREFETCH_L1(tag_bucket);
if (self->block_bits_ > 4) PREFETCH_L1(bucket + 16);
out->len = 0;
out->len_code_delta = 0;
BROTLI_DCHECK(cur_ix_masked + max_length <= ring_buffer_mask + 1);
/* Try last distance first. */
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
const size_t backward = (size_t)distance_cache[i];
size_t prev_ix = (size_t)(cur_ix - backward);
if (prev_ix >= cur_ix) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
continue;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 3 || (len == 2 && i < 2)) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
if (i != 0) score -= BackwardReferencePenaltyUsingLastDistance(i);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
}
/* we require matches of len >4, so increase best_len to 3, so we can compare
* 4 bytes all the time. */
if (best_len < 3) {
best_len = 3;
}
{
const uint8_t tag = hash & TAG_HASH_MASK;
const size_t head = (num[key] + 1) & self->block_mask_;
uint64_t matches =
GetMatchingTagMask(self->block_size_ / 16, tag, tag_bucket, head);
/* Mask off any matches from uninitialized tags. */
uint16_t n = 65535 - num[key];
uint64_t block_has_unused_slots = self->block_size_ > n;
uint64_t mask = (block_has_unused_slots << (n & (64 - 1))) - 1;
matches &= mask;
for (; matches > 0; matches &= (matches - 1)) {
const size_t rb_index =
(head + (size_t)BROTLI_TZCNT64(matches)) & self->block_mask_;
size_t prev_ix = bucket[rb_index];
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (cur_ix_masked + best_len > ring_buffer_mask) {
break;
}
if (prev_ix + best_len > ring_buffer_mask ||
/* compare 4 bytes ending at best_len + 1 */
BrotliUnalignedRead32(&data[cur_ix_masked + best_len - 3]) !=
BrotliUnalignedRead32(&data[prev_ix + best_len - 3])) {
continue;
}
{
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
/* Comparing for >= 3 does not change the semantics, but just saves
for a few unnecessary binary logarithms in backward reference
score, since we are not interested in such short matches. */
score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = best_len;
out->distance = backward;
out->score = best_score;
}
}
}
}
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
tag_bucket[num[key] & self->block_mask_] = tag;
--num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary,
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
max_distance, out, BROTLI_FALSE);
}
}
#undef HashLongestMatch

View File

@@ -1,212 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, JUMP, NUMBUCKETS, MASK, CHUNKLEN */
/* NUMBUCKETS / (MASK + 1) = probability of storing and using hash code. */
/* JUMP = skip bytes for speedup */
/* Rolling hash for long distance long string matches. Stores one position
per bucket, bucket key is computed over a long region. */
#define HashRolling HASHER()
static const uint32_t FN(kRollingHashMul32) = 69069;
static const uint32_t FN(kInvalidPos) = 0xffffffff;
/* This hasher uses a longer forward length, but returning a higher value here
will hurt compression by the main hasher when combined with a composite
hasher. The hasher tests for forward itself instead. */
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* Computes a code from a single byte. A lookup table of 256 values could be
used, but simply adding 1 works about as good. */
static uint32_t FN(HashByte)(uint8_t byte) {
return (uint32_t)byte + 1u;
}
static uint32_t FN(HashRollingFunctionInitial)(uint32_t state, uint8_t add,
uint32_t factor) {
return (uint32_t)(factor * state + FN(HashByte)(add));
}
static uint32_t FN(HashRollingFunction)(uint32_t state, uint8_t add,
uint8_t rem, uint32_t factor,
uint32_t factor_remove) {
return (uint32_t)(factor * state +
FN(HashByte)(add) - factor_remove * FN(HashByte)(rem));
}
typedef struct HashRolling {
uint32_t state;
uint32_t* table;
size_t next_ix;
uint32_t chunk_len;
uint32_t factor;
uint32_t factor_remove;
} HashRolling;
static void FN(Initialize)(
HasherCommon* common, HashRolling* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
size_t i;
self->state = 0;
self->next_ix = 0;
self->factor = FN(kRollingHashMul32);
/* Compute the factor of the oldest byte to remove: factor**steps modulo
0xffffffff (the multiplications rely on 32-bit overflow) */
self->factor_remove = 1;
for (i = 0; i < CHUNKLEN; i += JUMP) {
self->factor_remove *= self->factor;
}
self->table = (uint32_t*)common->extra[0];
for (i = 0; i < NUMBUCKETS; i++) {
self->table[i] = FN(kInvalidPos);
}
BROTLI_UNUSED(params);
}
static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
size_t i;
/* Too small size, cannot use this hasher. */
if (input_size < CHUNKLEN) return;
self->state = 0;
for (i = 0; i < CHUNKLEN; i += JUMP) {
self->state = FN(HashRollingFunctionInitial)(
self->state, data[i], self->factor);
}
BROTLI_UNUSED(one_shot);
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = NUMBUCKETS * sizeof(uint32_t);
}
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
BROTLI_UNUSED(self);
BROTLI_UNUSED(data);
BROTLI_UNUSED(mask);
BROTLI_UNUSED(ix);
}
static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
BROTLI_UNUSED(self);
BROTLI_UNUSED(data);
BROTLI_UNUSED(mask);
BROTLI_UNUSED(ix_start);
BROTLI_UNUSED(ix_end);
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashRolling* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ring_buffer_mask) {
/* In this case we must re-initialize the hasher from scratch from the
current position. */
size_t position_masked;
size_t available = num_bytes;
if ((position & (JUMP - 1)) != 0) {
size_t diff = JUMP - (position & (JUMP - 1));
available = (diff > available) ? 0 : (available - diff);
position += diff;
}
position_masked = position & ring_buffer_mask;
/* wrapping around ringbuffer not handled. */
if (available > ring_buffer_mask - position_masked) {
available = ring_buffer_mask - position_masked;
}
FN(Prepare)(self, BROTLI_FALSE, available,
ringbuffer + (position & ring_buffer_mask));
self->next_ix = position;
BROTLI_UNUSED(num_bytes);
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HashRolling* BROTLI_RESTRICT self,
int* BROTLI_RESTRICT distance_cache) {
BROTLI_UNUSED(self);
BROTLI_UNUSED(distance_cache);
}
static BROTLI_INLINE void FN(FindLongestMatch)(
HashRolling* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t pos;
if ((cur_ix & (JUMP - 1)) != 0) return;
/* Not enough lookahead */
if (max_length < CHUNKLEN) return;
for (pos = self->next_ix; pos <= cur_ix; pos += JUMP) {
uint32_t code = self->state & MASK;
uint8_t rem = data[pos & ring_buffer_mask];
uint8_t add = data[(pos + CHUNKLEN) & ring_buffer_mask];
size_t found_ix = FN(kInvalidPos);
self->state = FN(HashRollingFunction)(
self->state, add, rem, self->factor, self->factor_remove);
if (code < NUMBUCKETS) {
found_ix = self->table[code];
self->table[code] = (uint32_t)pos;
if (pos == cur_ix && found_ix != FN(kInvalidPos)) {
/* The cast to 32-bit makes backward distances up to 4GB work even
if cur_ix is above 4GB, despite using 32-bit values in the table. */
size_t backward = (uint32_t)(cur_ix - found_ix);
if (backward <= max_backward) {
const size_t found_ix_masked = found_ix & ring_buffer_mask;
const size_t len = FindMatchLengthWithLimit(&data[found_ix_masked],
&data[cur_ix_masked],
max_length);
if (len >= 4 && len > out->len) {
score_t score = BackwardReferenceScore(len, backward);
if (score > out->score) {
out->len = len;
out->distance = backward;
out->score = score;
out->len_code_delta = 0;
}
}
}
}
}
}
self->next_ix = cur_ix + JUMP;
/* NOTE: this hasher does not search in the dictionary. It is used as
backup-hasher, the main hasher already searches in it. */
BROTLI_UNUSED(dictionary);
BROTLI_UNUSED(distance_cache);
BROTLI_UNUSED(dictionary_distance);
BROTLI_UNUSED(max_distance);
}
#undef HashRolling

View File

@@ -1,330 +0,0 @@
/* NOLINT(build/header_guard) */
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, BUCKET_BITS, MAX_TREE_COMP_LENGTH,
MAX_TREE_SEARCH_DEPTH */
/* A (forgetful) hash table where each hash bucket contains a binary tree of
sequences whose first 4 bytes share the same hash code.
Each sequence is MAX_TREE_COMP_LENGTH long and is identified by its starting
position in the input data. The binary tree is sorted by the lexicographic
order of the sequences, and it is also a max-heap with respect to the
starting positions. */
#define HashToBinaryTree HASHER()
#define BUCKET_SIZE (1 << BUCKET_BITS)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
return MAX_TREE_COMP_LENGTH;
}
static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - BUCKET_BITS);
}
typedef struct HashToBinaryTree {
/* The window size minus 1 */
size_t window_mask_;
/* Hash table that maps the 4-byte hashes of the sequence to the last
position where this hash was found, which is the root of the binary
tree of sequences that share this hash bucket. */
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
/* A position used to mark a non-existent sequence, i.e. a tree is empty if
its root is at invalid_pos_ and a node is a leaf if both its children
are at invalid_pos_. */
uint32_t invalid_pos_;
/* --- Dynamic size members --- */
/* The union of the binary trees of each hash bucket. The root of the tree
corresponding to a hash is a sequence starting at buckets_[hash] and
the left and right children of a sequence starting at pos are
forest_[2 * pos] and forest_[2 * pos + 1]. */
uint32_t* forest_; /* uint32_t[2 * num_nodes] */
} HashToBinaryTree;
static void FN(Initialize)(
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->buckets_ = (uint32_t*)common->extra[0];
self->forest_ = (uint32_t*)common->extra[1];
self->window_mask_ = (1u << params->lgwin) - 1u;
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
}
static void FN(Prepare)
(HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint32_t invalid_pos = self->invalid_pos_;
uint32_t i;
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
BROTLI_UNUSED(data);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
for (i = 0; i < BUCKET_SIZE; i++) {
buckets[i] = invalid_pos;
}
}
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size, size_t* alloc_size) {
size_t num_nodes = (size_t)1 << params->lgwin;
if (one_shot && input_size < num_nodes) {
num_nodes = input_size;
}
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
alloc_size[1] = 2 * sizeof(uint32_t) * num_nodes;
}
static BROTLI_INLINE size_t FN(LeftChildIndex)(
HashToBinaryTree* BROTLI_RESTRICT self,
const size_t pos) {
return 2 * (pos & self->window_mask_);
}
static BROTLI_INLINE size_t FN(RightChildIndex)(
HashToBinaryTree* BROTLI_RESTRICT self,
const size_t pos) {
return 2 * (pos & self->window_mask_) + 1;
}
/* Stores the hash of the next 4 bytes and in a single tree-traversal, the
hash bucket's binary tree is searched for matches and is re-rooted at the
current position.
If less than MAX_TREE_COMP_LENGTH data is available, the hash bucket of the
current position is searched for matches, but the state of the hash table
is not changed, since we can not know the final sorting order of the
current (incomplete) sequence.
This function must be called with increasing cur_ix positions. */
static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
BackwardMatch* BROTLI_RESTRICT matches) {
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
const size_t max_comp_len =
BROTLI_MIN(size_t, max_length, MAX_TREE_COMP_LENGTH);
const BROTLI_BOOL should_reroot_tree =
TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
uint32_t* BROTLI_RESTRICT forest = self->forest_;
size_t prev_ix = buckets[key];
/* The forest index of the rightmost node of the left subtree of the new
root, updated as we traverse and re-root the tree of the hash bucket. */
size_t node_left = FN(LeftChildIndex)(self, cur_ix);
/* The forest index of the leftmost node of the right subtree of the new
root, updated as we traverse and re-root the tree of the hash bucket. */
size_t node_right = FN(RightChildIndex)(self, cur_ix);
/* The match length of the rightmost node of the left subtree of the new
root, updated as we traverse and re-root the tree of the hash bucket. */
size_t best_len_left = 0;
/* The match length of the leftmost node of the right subtree of the new
root, updated as we traverse and re-root the tree of the hash bucket. */
size_t best_len_right = 0;
size_t depth_remaining;
if (should_reroot_tree) {
buckets[key] = (uint32_t)cur_ix;
}
for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
const size_t backward = cur_ix - prev_ix;
const size_t prev_ix_masked = prev_ix & ring_buffer_mask;
if (backward == 0 || backward > max_backward || depth_remaining == 0) {
if (should_reroot_tree) {
forest[node_left] = self->invalid_pos_;
forest[node_right] = self->invalid_pos_;
}
break;
}
{
const size_t cur_len = BROTLI_MIN(size_t, best_len_left, best_len_right);
size_t len;
BROTLI_DCHECK(cur_len <= MAX_TREE_COMP_LENGTH);
len = cur_len +
FindMatchLengthWithLimit(&data[cur_ix_masked + cur_len],
&data[prev_ix_masked + cur_len],
max_length - cur_len);
BROTLI_DCHECK(
0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
if (matches && len > *best_len) {
*best_len = len;
InitBackwardMatch(matches++, backward, len);
}
if (len >= max_comp_len) {
if (should_reroot_tree) {
forest[node_left] = forest[FN(LeftChildIndex)(self, prev_ix)];
forest[node_right] = forest[FN(RightChildIndex)(self, prev_ix)];
}
break;
}
if (data[cur_ix_masked + len] > data[prev_ix_masked + len]) {
best_len_left = len;
if (should_reroot_tree) {
forest[node_left] = (uint32_t)prev_ix;
}
node_left = FN(RightChildIndex)(self, prev_ix);
prev_ix = forest[node_left];
} else {
best_len_right = len;
if (should_reroot_tree) {
forest[node_right] = (uint32_t)prev_ix;
}
node_right = FN(LeftChildIndex)(self, prev_ix);
prev_ix = forest[node_right];
}
}
}
return matches;
}
/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the
length of max_length and stores the position cur_ix in the hash table.
Sets *num_matches to the number of matches found, and stores the found
matches in matches[0] to matches[*num_matches - 1]. The matches will be
sorted by strictly increasing length and (non-strictly) increasing
distance. */
static BROTLI_INLINE size_t FN(FindAllMatches)(
HashToBinaryTree* BROTLI_RESTRICT self,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const BrotliEncoderParams* params,
BackwardMatch* matches) {
BackwardMatch* const orig_matches = matches;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t best_len = 1;
const size_t short_match_max_backward =
params->quality != HQ_ZOPFLIFICATION_QUALITY ? 16 : 64;
size_t stop = cur_ix - short_match_max_backward;
uint32_t dict_matches[BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1];
size_t i;
if (cur_ix < short_match_max_backward) { stop = 0; }
for (i = cur_ix - 1; i > stop && best_len <= 2; --i) {
size_t prev_ix = i;
const size_t backward = cur_ix - prev_ix;
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (data[cur_ix_masked] != data[prev_ix] ||
data[cur_ix_masked + 1] != data[prev_ix + 1]) {
continue;
}
{
const size_t len =
FindMatchLengthWithLimit(&data[prev_ix], &data[cur_ix_masked],
max_length);
if (len > best_len) {
best_len = len;
InitBackwardMatch(matches++, backward, len);
}
}
}
if (best_len < max_length) {
matches = FN(StoreAndFindMatches)(self, data, cur_ix,
ring_buffer_mask, max_length, max_backward, &best_len, matches);
}
for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
dict_matches[i] = kInvalidMatch;
}
{
size_t minlen = BROTLI_MAX(size_t, 4, best_len + 1);
if (BrotliFindAllStaticDictionaryMatches(dictionary,
&data[cur_ix_masked], minlen, max_length, &dict_matches[0])) {
size_t maxlen = BROTLI_MIN(
size_t, BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN, max_length);
size_t l;
for (l = minlen; l <= maxlen; ++l) {
uint32_t dict_id = dict_matches[l];
if (dict_id < kInvalidMatch) {
size_t distance = dictionary_distance + (dict_id >> 5) + 1;
if (distance <= params->dist.max_distance) {
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
}
}
}
}
}
return (size_t)(matches - orig_matches);
}
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
current sequence, without returning any matches.
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data,
const size_t mask, const size_t ix) {
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
max_backward, NULL, NULL);
}
static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
const size_t ix_start, const size_t ix_end) {
size_t i = ix_start;
size_t j = ix_start;
if (ix_start + 63 <= ix_end) {
i = ix_end - 63;
}
if (ix_start + 512 <= i) {
for (; j < i; j += 8) {
FN(Store)(self, data, mask, j);
}
}
for (; i < ix_end; ++i) {
FN(Store)(self, data, mask, i);
}
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
HashToBinaryTree* BROTLI_RESTRICT self,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
if (num_bytes >= FN(HashTypeLength)() - 1 &&
position >= MAX_TREE_COMP_LENGTH) {
/* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.
These could not be calculated before, since they require knowledge
of both the previous and the current block. */
const size_t i_start = position - MAX_TREE_COMP_LENGTH + 1;
const size_t i_end = BROTLI_MIN(size_t, position, i_start + num_bytes);
size_t i;
for (i = i_start; i < i_end; ++i) {
/* Maximum distance is window size - 16, see section 9.1. of the spec.
Furthermore, we have to make sure that we don't look further back
from the start of the next block than the window size, otherwise we
could access already overwritten areas of the ring-buffer. */
const size_t max_backward =
self->window_mask_ - BROTLI_MAX(size_t,
BROTLI_WINDOW_GAP - 1,
position - i);
/* We know that i + MAX_TREE_COMP_LENGTH <= position + num_bytes, i.e. the
end of the current block and that we have at least
MAX_TREE_COMP_LENGTH tail in the ring-buffer. */
FN(StoreAndFindMatches)(self, ringbuffer, i, ringbuffer_mask,
MAX_TREE_COMP_LENGTH, max_backward, NULL, NULL);
}
}
}
#undef BUCKET_SIZE
#undef HashToBinaryTree

View File

@@ -1,69 +0,0 @@
#ifndef THIRD_PARTY_BROTLI_ENC_MATCHING_TAG_MASK_H_
#define THIRD_PARTY_BROTLI_ENC_MATCHING_TAG_MASK_H_
#include "../common/platform.h"
#if defined(__SSE2__) || defined(_M_AMD64) || \
(defined(_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
#define SUPPORTS_SSE_2
#endif
#if defined(SUPPORTS_SSE_2)
#include <immintrin.h>
#endif
static BROTLI_INLINE uint64_t GetMatchingTagMask(
size_t chunk_count, const uint8_t tag,
const uint8_t* BROTLI_RESTRICT tag_bucket, const size_t head) {
uint64_t matches = 0;
#if defined(SUPPORTS_SSE_2)
const __m128i comparison_mask = _mm_set1_epi8((char)tag);
size_t i;
for (i = 0; i < chunk_count && i < 4; i++) {
const __m128i chunk =
_mm_loadu_si128((const __m128i*)(const void*)(tag_bucket + 16 * i));
const __m128i equal_mask = _mm_cmpeq_epi8(chunk, comparison_mask);
matches |= (uint64_t)_mm_movemask_epi8(equal_mask) << 16 * i;
}
#else
const int chunk_size = sizeof(size_t);
const size_t shift_amount = ((chunk_size * 8) - chunk_size);
const size_t xFF = ~((size_t)0);
const size_t x01 = xFF / 0xFF;
const size_t x80 = x01 << 7;
const size_t splat_char = tag * x01;
int i = ((int)chunk_count * 16) - chunk_size;
BROTLI_DCHECK((sizeof(size_t) == 4) || (sizeof(size_t) == 8));
#if BROTLI_LITTLE_ENDIAN
const size_t extractMagic = (xFF / 0x7F) >> chunk_size;
do {
size_t chunk = BrotliUnalignedReadSizeT(&tag_bucket[i]);
chunk ^= splat_char;
chunk = (((chunk | x80) - x01) | chunk) & x80;
matches <<= chunk_size;
matches |= (chunk * extractMagic) >> shift_amount;
i -= chunk_size;
} while (i >= 0);
#else
const size_t msb = xFF ^ (xFF >> 1);
const size_t extractMagic = (msb / 0x1FF) | msb;
do {
size_t chunk = BrotliUnalignedReadSizeT(&tag_bucket[i]);
chunk ^= splat_char;
chunk = (((chunk | x80) - x01) | chunk) & x80;
matches <<= chunk_size;
matches |= ((chunk >> 7) * extractMagic) >> shift_amount;
i -= chunk_size;
} while (i >= 0);
#endif
matches = ~matches;
#endif
if (chunk_count == 1) return BrotliRotateRight16((uint16_t)matches, head);
if (chunk_count == 2) return BrotliRotateRight32((uint32_t)matches, head);
return BrotliRotateRight64(matches, head);
}
#undef SUPPORTS_SSE_2
#endif // THIRD_PARTY_BROTLI_ENC_MATCHING_TAG_MASK_H_

View File

@@ -1,127 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Macros for memory management. */
#ifndef BROTLI_ENC_MEMORY_H_
#define BROTLI_ENC_MEMORY_H_
#include "../common/platform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if !defined(BROTLI_ENCODER_CLEANUP_ON_OOM) && \
!defined(BROTLI_ENCODER_EXIT_ON_OOM)
#define BROTLI_ENCODER_EXIT_ON_OOM
#endif
#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
#if defined(BROTLI_EXPERIMENTAL)
#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS (48*1024)
#else /* BROTLI_EXPERIMENTAL */
#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256
#endif /* BROTLI_EXPERIMENTAL */
#else /* BROTLI_ENCODER_EXIT_ON_OOM */
#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 0
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
typedef struct MemoryManager {
brotli_alloc_func alloc_func;
brotli_free_func free_func;
void* opaque;
#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
BROTLI_BOOL is_oom;
size_t perm_allocated;
size_t new_allocated;
size_t new_freed;
void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS];
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
} MemoryManager;
BROTLI_INTERNAL void BrotliInitMemoryManager(
MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
void* opaque);
BROTLI_INTERNAL void* BrotliAllocate(MemoryManager* m, size_t n);
#define BROTLI_ALLOC(M, T, N) \
((N) > 0 ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
#define BROTLI_FREE(M, P) { \
BrotliFree((M), (P)); \
P = NULL; \
}
#if defined(BROTLI_ENCODER_EXIT_ON_OOM)
#define BROTLI_IS_OOM(M) (!!0)
#else /* BROTLI_ENCODER_EXIT_ON_OOM */
#define BROTLI_IS_OOM(M) (!!(M)->is_oom)
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
/*
BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
The only purpose of it is to explain static analyzers the state of things.
NB: use ONLY together with BROTLI_IS_OOM
AND ONLY for allocations in the current scope.
*/
#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
#define BROTLI_IS_NULL(A) ((A) == nullptr)
#else /* defined(__clang_analyzer__) */
#define BROTLI_IS_NULL(A) (!!0)
#endif /* defined(__clang_analyzer__) */
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
/*
Dynamically grows array capacity to at least the requested size
M: MemoryManager
T: data type
A: array
C: capacity
R: requested size
*/
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
if (C < (R)) { \
size_t _new_size = (C == 0) ? (R) : C; \
T* new_array; \
while (_new_size < (R)) _new_size *= 2; \
new_array = BROTLI_ALLOC((M), T, _new_size); \
if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
memcpy(new_array, A, C * sizeof(T)); \
BROTLI_FREE((M), A); \
A = new_array; \
C = _new_size; \
} \
}
/*
Appends value and dynamically grows array capacity when needed
M: MemoryManager
T: data type
A: array
C: array capacity
S: array size
V: value to append
*/
#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
(S)++; \
BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \
A[(S) - 1] = (V); \
}
/* "Bootstrap" allocations are not tracked by memory manager; should be used
only to allocate MemoryManager itself (or structure containing it). */
BROTLI_INTERNAL void* BrotliBootstrapAlloc(size_t size,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliBootstrapFree(void* address, MemoryManager* m);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_MEMORY_H_ */

View File

@@ -1,46 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Parameters for the Brotli encoder with chosen quality levels. */
#ifndef BROTLI_ENC_PARAMS_H_
#define BROTLI_ENC_PARAMS_H_
#include <brotli/encode.h>
#include "encoder_dict.h"
typedef struct BrotliHasherParams {
int type;
int bucket_bits;
int block_bits;
int num_last_distances_to_check;
} BrotliHasherParams;
typedef struct BrotliDistanceParams {
uint32_t distance_postfix_bits;
uint32_t num_direct_distance_codes;
uint32_t alphabet_size_max;
uint32_t alphabet_size_limit;
size_t max_distance;
} BrotliDistanceParams;
/* Encoding parameters */
typedef struct BrotliEncoderParams {
BrotliEncoderMode mode;
int quality;
int lgwin;
int lgblock;
size_t stream_offset;
size_t size_hint;
BROTLI_BOOL disable_literal_context_modeling;
BROTLI_BOOL large_window;
BrotliHasherParams hasher;
BrotliDistanceParams dist;
/* TODO(eustas): rename to BrotliShared... */
SharedEncoderDictionary dictionary;
} BrotliEncoderParams;
#endif /* BROTLI_ENC_PARAMS_H_ */

View File

@@ -1,209 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Constants and formulas that affect speed-ratio trade-offs and thus define
quality levels. */
#ifndef BROTLI_ENC_QUALITY_H_
#define BROTLI_ENC_QUALITY_H_
#include "../common/platform.h"
#include <brotli/encode.h>
#include "params.h"
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
#define ZOPFLIFICATION_QUALITY 10
#define HQ_ZOPFLIFICATION_QUALITY 11
#define MAX_QUALITY_FOR_STATIC_ENTROPY_CODES 2
#define MIN_QUALITY_FOR_BLOCK_SPLIT 4
#define MIN_QUALITY_FOR_NONZERO_DISTANCE_PARAMS 4
#define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
#define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
#define MIN_QUALITY_FOR_CONTEXT_MODELING 5
#define MIN_QUALITY_FOR_HQ_CONTEXT_MODELING 7
#define MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING 10
/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting,
so we buffer at most this much literals and commands. */
#define MAX_NUM_DELAYED_SYMBOLS 0x2FFF
/* Returns hash-table size for quality levels 0 and 1. */
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
}
/* The maximum length for which the zopflification uses distinct distances. */
#define MAX_ZOPFLI_LEN_QUALITY_10 150
#define MAX_ZOPFLI_LEN_QUALITY_11 325
/* Do not thoroughly search when a long copy is found. */
#define BROTLI_LONG_COPY_QUICK_STEP 16384
static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
return params->quality <= 10 ?
MAX_ZOPFLI_LEN_QUALITY_10 :
MAX_ZOPFLI_LEN_QUALITY_11;
}
/* Number of best candidates to evaluate to expand Zopfli chain. */
static BROTLI_INLINE size_t MaxZopfliCandidates(
const BrotliEncoderParams* params) {
return params->quality <= 10 ? 1 : 5;
}
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
if (params->quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES) {
params->large_window = BROTLI_FALSE;
}
if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
params->lgwin = BROTLI_MIN_WINDOW_BITS;
} else {
int max_lgwin = params->large_window ? BROTLI_LARGE_MAX_WINDOW_BITS :
BROTLI_MAX_WINDOW_BITS;
if (params->lgwin > max_lgwin) params->lgwin = max_lgwin;
}
}
/* Returns optimized lg_block value. */
static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
int lgblock = params->lgblock;
if (params->quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
params->quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
lgblock = params->lgwin;
} else if (params->quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
lgblock = 14;
} else if (lgblock == 0) {
lgblock = 16;
if (params->quality >= 9 && params->lgwin > lgblock) {
lgblock = BROTLI_MIN(int, 18, params->lgwin);
}
} else {
lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS,
BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock));
}
return lgblock;
}
/* Returns log2 of the size of main ring buffer area.
Allocate at least lgwin + 1 bits for the ring buffer so that the newly
added block fits there completely and we still get lgwin bits and at least
read_block_size_bits + 1 bits because the copy tail length needs to be
smaller than ring-buffer size. */
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
}
static BROTLI_INLINE size_t MaxMetablockSize(
const BrotliEncoderParams* params) {
int bits =
BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS);
return (size_t)1 << bits;
}
/* When searching for backward references and have not seen matches for a long
time, we can skip some match lookups. Unsuccessful match lookups are very
expensive and this kind of a heuristic speeds up compression quite a lot.
At first 8 byte strides are taken and every second byte is put to hasher.
After 4x more literals stride by 16 bytes, every put 4-th byte to hasher.
Applied only to qualities 2 to 9. */
static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
const BrotliEncoderParams* params) {
return params->quality < 9 ? 64 : 512;
}
/* Quality to hasher mapping:
- q02: h02 (longest_match_quickly), b16, l5
- q03: h03 (longest_match_quickly), b17, l5
- q04: h04 (longest_match_quickly), b17, l5
- q04: h54 (longest_match_quickly), b20, l7 | for large files
- q05: h58 (longest_match_simd ), b14, l4
- q05: h68 (longest_match64_simd ), b15, l5 | for large files
- q05: h40 (forgetful_chain ), b15, l4 | for small window
- q06: h58 (longest_match_simd ), b14, l4
- q06: h68 (longest_match64_simd ), b15, l5 | for large files
- q06: h40 (forgetful_chain ), b15, l4 | for small window
- q07: h58 (longest_match_simd ), b15, l4
- q07: h68 (longest_match64_simd ), b15, l5 | for large files
- q07: h41 (forgetful_chain ), b15, l4 | for small window
- q08: h05 (longest_match ), b15, l4
- q08: h06 (longest_match64 ), b15, l5 | for large files
- q08: h41 (forgetful_chain ), b15, l4 | for small window
- q09: h05 (longest_match ), b15, l4
- q09: h06 (longest_match64 ), b15, l5 | for large files
- q09: h42 (forgetful_chain ), b15, l4 | for small window
- q10: t10 (to_binary_tree ), b17, l128
- q11: t10 (to_binary_tree ), b17, l128
Where "q" is quality, "h" is hasher type, "b" is bucket bits,
"l" is source len. */
static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
BrotliHasherParams* hparams) {
if (params->quality > 9) {
hparams->type = 10;
} else if (params->quality == 4 && params->size_hint >= (1 << 20)) {
hparams->type = 54;
} else if (params->quality < 5) {
hparams->type = params->quality;
} else if (params->lgwin <= 16) {
hparams->type = params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42;
} else if (params->size_hint >= (1 << 20) && params->lgwin >= 19) {
#if defined(BROTLI_MAX_SIMD_QUALITY)
hparams->type = params->quality <= BROTLI_MAX_SIMD_QUALITY ? 68 : 6;
#else
hparams->type = 6;
#endif
hparams->block_bits = params->quality - 1;
hparams->bucket_bits = 15;
hparams->num_last_distances_to_check =
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
} else {
/* TODO(eustas): often previous setting (H6) is faster and denser; consider
adding an option to use it. */
#if defined(BROTLI_MAX_SIMD_QUALITY)
hparams->type = params->quality <= BROTLI_MAX_SIMD_QUALITY ? 58 : 5;
#else
hparams->type = 5;
#endif
hparams->block_bits = params->quality - 1;
hparams->bucket_bits = params->quality < 7 ? 14 : 15;
hparams->num_last_distances_to_check =
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
}
if (params->lgwin > 24) {
/* Different hashers for large window brotli: not for qualities <= 2,
these are too fast for large window. Not for qualities >= 10: their
hasher already works well with large window. So the changes are:
H3 --> H35: for quality 3.
H54 --> H55: for quality 4 with size hint > 1MB
H6/H68 --> H65: for qualities 5, 6, 7, 8, 9. */
if (hparams->type == 3) {
hparams->type = 35;
}
if (hparams->type == 54) {
hparams->type = 55;
}
if (hparams->type == 6 || hparams->type == 68) {
hparams->type = 65;
}
}
}
#endif /* BROTLI_ENC_QUALITY_H_ */

View File

@@ -1,104 +0,0 @@
/* Copyright 2022 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Encoder state. */
#ifndef BROTLI_ENC_STATE_H_
#define BROTLI_ENC_STATE_H_
#include "../common/constants.h"
#include "../common/platform.h"
#include "command.h"
#include "compress_fragment.h"
#include "compress_fragment_two_pass.h"
#include "hash.h"
#include "memory.h"
#include "params.h"
#include "ringbuffer.h"
typedef enum BrotliEncoderStreamState {
/* Default state. */
BROTLI_STREAM_PROCESSING = 0,
/* Intermediate state; after next block is emitted, byte-padding should be
performed before getting back to default state. */
BROTLI_STREAM_FLUSH_REQUESTED = 1,
/* Last metablock was produced; no more input is acceptable. */
BROTLI_STREAM_FINISHED = 2,
/* Flushing compressed block and writing meta-data block header. */
BROTLI_STREAM_METADATA_HEAD = 3,
/* Writing metadata block body. */
BROTLI_STREAM_METADATA_BODY = 4
} BrotliEncoderStreamState;
typedef enum BrotliEncoderFlintState {
BROTLI_FLINT_NEEDS_2_BYTES = 2,
BROTLI_FLINT_NEEDS_1_BYTE = 1,
BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
BROTLI_FLINT_DONE = -2
} BrotliEncoderFlintState;
typedef struct BrotliEncoderStateStruct {
BrotliEncoderParams params;
MemoryManager memory_manager_;
uint64_t input_pos_;
RingBuffer ringbuffer_;
size_t cmd_alloc_size_;
Command* commands_;
size_t num_commands_;
size_t num_literals_;
size_t last_insert_len_;
uint64_t last_flush_pos_;
uint64_t last_processed_pos_;
int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
int saved_dist_cache_[4];
uint16_t last_bytes_;
uint8_t last_bytes_bits_;
/* "Flint" is a tiny uncompressed block emitted before the continuation
block to unwire literal context from previous data. Despite being int8_t,
field is actually BrotliEncoderFlintState enum. */
int8_t flint_;
uint8_t prev_byte_;
uint8_t prev_byte2_;
size_t storage_size_;
uint8_t* storage_;
Hasher hasher_;
/* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
int small_table_[1 << 10]; /* 4KiB */
int* large_table_; /* Allocated only when needed */
size_t large_table_size_;
BrotliOnePassArena* one_pass_arena_;
BrotliTwoPassArena* two_pass_arena_;
/* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
uint32_t* command_buf_;
uint8_t* literal_buf_;
uint64_t total_in_;
uint8_t* next_out_;
size_t available_out_;
uint64_t total_out_;
/* Temporary buffer for padding flush bits or metadata block header / body. */
union {
uint64_t u64[2];
uint8_t u8[16];
} tiny_buf_;
uint32_t remaining_metadata_bytes_;
BrotliEncoderStreamState stream_state_;
BROTLI_BOOL is_last_block_emitted_;
BROTLI_BOOL is_initialized_;
} BrotliEncoderStateStruct;
typedef struct BrotliEncoderStateStruct BrotliEncoderStateInternal;
#define BrotliEncoderState BrotliEncoderStateInternal
#endif // BROTLI_ENC_STATE_H_

View File

@@ -1,224 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup table for static dictionary and transforms. */
#include "static_dict_lut.h"
#include "../common/platform.h" /* IWYU pragma: keep */
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/dictionary.h"
#include "../common/transform.h"
#include "hash_base.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
/* TODO(eustas): deal with largest bucket(s). Not it contains 163 items. */
static BROTLI_BOOL BROTLI_COLD DoBrotliEncoderInitStaticDictionaryLut(
const BrotliDictionary* dict, uint16_t* buckets, DictWord* words,
void* arena) {
DictWord* slots = (DictWord*)arena;
uint16_t* heads = (uint16_t*)(slots + BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS);
uint16_t* counts = heads + BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS;
uint16_t* prev = counts + BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS;
size_t next_slot = 0;
uint8_t transformed_word[24];
uint8_t transformed_other_word[24];
size_t l;
size_t pos;
size_t i;
memset(counts, 0, BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS * sizeof(uint16_t));
memset(heads, 0, BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS * sizeof(uint16_t));
memset(prev, 0, BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS * sizeof(uint16_t));
for (l = 4; l <= 24; ++l) {
size_t n = 1u << dict->size_bits_by_length[l];
const uint8_t* dict_words = dict->data + dict->offsets_by_length[l];
for (i = 0; i < n; ++i) {
const uint8_t* dict_word = dict_words + l * i;
uint32_t key = Hash15(dict_word);
slots[next_slot].len = (uint8_t)l;
slots[next_slot].transform = BROTLI_TRANSFORM_IDENTITY;
slots[next_slot].idx = (uint16_t)i;
prev[next_slot] = heads[key];
heads[key] = (uint16_t)next_slot;
counts[key]++;
++next_slot;
}
for (i = 0; i < n; ++i) {
uint32_t key;
uint32_t prefix;
BROTLI_BOOL found;
size_t curr;
const uint8_t* dict_word = dict_words + l * i;
if (dict_word[0] < 'a' || dict_word[0] > 'z') continue;
memcpy(transformed_word, dict_word, l);
transformed_word[0] = transformed_word[0] - 32;
key = Hash15(transformed_word);
prefix = BROTLI_UNALIGNED_LOAD32LE(transformed_word) & ~0x20202020u;
found = BROTLI_FALSE;
curr = heads[key];
while (curr != 0) {
const uint8_t* other_word;
uint32_t other_prefix;
if (slots[curr].len != l) break;
other_word = dict_words + l * slots[curr].idx;
other_prefix = BROTLI_UNALIGNED_LOAD32LE(other_word) & ~0x20202020u;
if (prefix == other_prefix) {
if (memcmp(transformed_word, other_word, l) == 0) {
found = BROTLI_TRUE;
break;
}
}
curr = prev[curr];
}
if (found) continue;
slots[next_slot].len = (uint8_t)l;
slots[next_slot].transform = BROTLI_TRANSFORM_UPPERCASE_FIRST;
slots[next_slot].idx = (uint16_t)i;
prev[next_slot] = heads[key];
heads[key] = (uint16_t)next_slot;
counts[key]++;
++next_slot;
}
for (i = 0; i < n; ++i) {
const uint8_t* dict_word = dict_words + l * i;
BROTLI_BOOL is_ascii = BROTLI_TRUE;
BROTLI_BOOL has_lower = BROTLI_FALSE;
size_t k;
uint32_t prefix;
uint32_t key;
size_t curr;
BROTLI_BOOL found;
for (k = 0; k < l; ++k) {
if (dict_word[k] >= 128) is_ascii = BROTLI_FALSE;
if (k > 0 && dict_word[k] >= 'a' && dict_word[k] <= 'z')
has_lower = BROTLI_TRUE;
}
if (!is_ascii || !has_lower) continue;
memcpy(transformed_word, dict_word, l);
prefix = BROTLI_UNALIGNED_LOAD32LE(transformed_word) & ~0x20202020u;
for (k = 0; k < l; ++k) {
if (transformed_word[k] >= 'a' && transformed_word[k] <= 'z') {
transformed_word[k] = transformed_word[k] - 32;
}
}
key = Hash15(transformed_word);
found = BROTLI_FALSE;
curr = heads[key];
while (curr != 0) {
const uint8_t* other_word;
uint32_t other_prefix;
if (slots[curr].len != l) break;
other_word = dict_words + l * slots[curr].idx;
other_prefix = BROTLI_UNALIGNED_LOAD32LE(other_word) & ~0x20202020u;
if (prefix == other_prefix) {
if (slots[curr].transform == BROTLI_TRANSFORM_IDENTITY) {
if (memcmp(transformed_word, other_word, l) == 0) {
found = BROTLI_TRUE;
break;
}
} else if (slots[curr].transform ==
BROTLI_TRANSFORM_UPPERCASE_FIRST) {
if ((transformed_word[0] == (other_word[0] - 32)) &&
memcmp(transformed_word + 1, other_word + 1, l - 1) == 0) {
found = BROTLI_TRUE;
break;
}
} else {
for (k = 0; k < l; ++k) {
if (other_word[k] >= 'a' && other_word[k] <= 'z') {
transformed_other_word[k] = other_word[k] - 32;
} else {
transformed_other_word[k] = other_word[k];
}
}
if (memcmp(transformed_word, transformed_other_word, l) == 0) {
found = BROTLI_TRUE;
break;
}
}
}
curr = prev[curr];
}
if (found) {
continue;
}
slots[next_slot].len = (uint8_t)l;
slots[next_slot].transform = BROTLI_TRANSFORM_UPPERCASE_ALL;
slots[next_slot].idx = (uint16_t)i;
prev[next_slot] = heads[key];
heads[key] = (uint16_t)next_slot;
counts[key]++;
++next_slot;
}
}
if (next_slot != 31704) return BROTLI_FALSE;
pos = 0;
/* Unused; makes offsets start from 1. */
words[pos].len = 0;
words[pos].transform = 0;
words[pos].idx = 0;
pos++;
for (i = 0; i < BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS; ++i) {
size_t num_words = counts[i];
size_t curr;
if (num_words == 0) {
buckets[i] = 0;
continue;
}
buckets[i] = (uint16_t)pos;
curr = heads[i];
pos += num_words;
for (size_t k = 0; k < num_words; ++k) {
words[pos - 1 - k] = slots[curr];
curr = prev[curr];
}
words[pos - 1].len |= 0x80;
}
return BROTLI_TRUE;
}
BROTLI_BOOL BrotliEncoderInitStaticDictionaryLut(
const BrotliDictionary* dict, uint16_t* buckets, DictWord* words) {
size_t arena_size =
BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS *
(sizeof(uint16_t) + sizeof(DictWord)) +
BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS * 2 * sizeof(uint16_t);
void* arena = malloc(arena_size);
BROTLI_BOOL ok;
if (arena == NULL) {
return BROTLI_FALSE;
}
ok = DoBrotliEncoderInitStaticDictionaryLut(dict, buckets, words, arena);
free(arena);
return ok;
}
BROTLI_MODEL("small")
uint16_t kStaticDictionaryBuckets[BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS];
BROTLI_MODEL("small")
DictWord kStaticDictionaryWords[BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS];
#else /* BROTLI_STATIC_INIT */
/* Embed kStaticDictionaryBuckets and kStaticDictionaryWords. */
#include "static_dict_lut_inc.h"
#endif /* BROTLI_STATIC_INIT */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,49 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup table for static dictionary and transforms. */
#ifndef BROTLI_ENC_STATIC_DICT_LUT_H_
#define BROTLI_ENC_STATIC_DICT_LUT_H_
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../common/static_init.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct DictWord {
/* Highest bit is used to indicate end of bucket. */
uint8_t len;
uint8_t transform;
uint16_t idx;
} DictWord;
/* Buckets are Hash15 results. */
#define BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS 32768
#define BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS 31705
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
BROTLI_INTERNAL BROTLI_BOOL BrotliEncoderInitStaticDictionaryLut(
const BrotliDictionary* dictionary, uint16_t* buckets, DictWord* words);
BROTLI_INTERNAL extern BROTLI_MODEL("small") uint16_t
kStaticDictionaryBuckets[BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS];
BROTLI_INTERNAL extern BROTLI_MODEL("small") DictWord
kStaticDictionaryWords[BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS];
#else
BROTLI_INTERNAL extern const BROTLI_MODEL("small") uint16_t
kStaticDictionaryBuckets[BROTLI_ENC_STATIC_DICT_LUT_NUM_BUCKETS];
BROTLI_INTERNAL extern const BROTLI_MODEL("small") DictWord
kStaticDictionaryWords[BROTLI_ENC_STATIC_DICT_LUT_NUM_ITEMS];
#endif
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_STATIC_DICT_LUT_H_ */

File diff suppressed because it is too large Load Diff

View File

@@ -1,59 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include "static_init.h"
#include "../common/platform.h"
#include "../common/static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
#include "../common/dictionary.h"
#include "dictionary_hash.h"
#include "static_dict_lut.h"
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_NONE)
static BROTLI_BOOL DoBrotliEncoderStaticInit(void) {
const BrotliDictionary* dict = BrotliGetDictionary();
BROTLI_BOOL ok = BrotliEncoderInitStaticDictionaryLut(
dict, kStaticDictionaryBuckets, kStaticDictionaryWords);
if (!ok) return BROTLI_FALSE;
ok = BrotliEncoderInitDictionaryHash(dict, kStaticDictionaryHashWords,
kStaticDictionaryHashLengths);
if (!ok) return BROTLI_FALSE;
return BROTLI_TRUE;
}
#endif /* BROTLI_STATIC_INIT_NONE */
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_EARLY)
static BROTLI_BOOL kEarlyInitOk;
static __attribute__((constructor)) void BrotliEncoderStaticInitEarly(void) {
kEarlyInitOk = DoBrotliEncoderStaticInit();
}
#elif (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_LAZY)
static BROTLI_BOOL kLazyInitOk;
void BrotliEncoderLazyStaticInitInner(void) {
kLazyInitOk = DoBrotliEncoderStaticInit();
}
#endif /* BROTLI_STATIC_INIT_EARLY */
BROTLI_BOOL BrotliEncoderEnsureStaticInit(void) {
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_NONE)
return BROTLI_TRUE;
#elif (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_EARLY)
return kEarlyInitOk;
#else
return kLazyInitOk;
#endif
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@@ -1,30 +0,0 @@
/* Copyright 2025 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Central point for static initialization. */
#ifndef THIRD_PARTY_BROTLI_ENC_STATIC_INIT_H_
#define THIRD_PARTY_BROTLI_ENC_STATIC_INIT_H_
#include "../common/platform.h"
#include "../common/static_init.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_STATIC_INIT == BROTLI_STATIC_INIT_LAZY)
BROTLI_INTERNAL void BrotliEncoderLazyStaticInitInner(void);
BROTLI_INTERNAL void BrotliEncoderLazyStaticInit(void);
#endif /* BROTLI_STATIC_INIT */
BROTLI_INTERNAL BROTLI_BOOL BrotliEncoderEnsureStaticInit(void);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif // THIRD_PARTY_BROTLI_ENC_STATIC_INIT_H_

View File

@@ -1,26 +0,0 @@
#include "../common/platform.h"
#include "../common/static_init.h"
#include "static_init.h"
#if (BROTLI_STATIC_INIT != BROTLI_STATIC_INIT_LAZY)
#error "BROTLI_STATIC_INIT should be BROTLI_STATIC_INIT_LAZY"
#else
void BrotliEncoderLazyStaticInit(void) {
/* From https://en.cppreference.com/w/cpp/language/storage_duration.html:
### Static block variables ###
Block variables with static or thread (since C++11) storage duration are
initialized the first time control passes through their declaration...
On all further calls, the declaration is skipped...
If multiple threads attempt to initialize the same static local variable
concurrently, the initialization occurs exactly once...
Usual implementations of this feature use variants of the double-checked
locking pattern, which reduces runtime overhead for already-initialized
local statics to a single non-atomic boolean comparison.
*/
static bool ok = [](){
BrotliEncoderLazyStaticInitInner();
return true;
}();
if (!ok) BROTLI_CRASH();
}
#endif /* BROTLI_STATIC_INIT_LAZY */

View File

@@ -1,18 +0,0 @@
# Force the use of Clang for C++ builds.
build --action_env=CC=clang
build --action_env=CXX=clang++
# Define the --config=asan-libfuzzer configuration.
build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan
# Define the --config=msan-libfuzzer configuration.
build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan
# Define the --config=ubsan-libfuzzer configuration.
build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=ubsan

View File

@@ -1,12 +0,0 @@
load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
# To start fuzzing run: bazel run --config=asan-libfuzzer //:fuzz_config_run
cc_fuzz_test(
name = "decode_fuzzer",
srcs = ["decode_fuzzer.c"],
deps = ["@org_brotli//:brotlidec"],
)

View File

@@ -1,20 +0,0 @@
# Copyright 2025 The Brotli Authors. All rights reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
"""Brotli reference implementation"""
module(
name = "brotli_fuzz",
version = "1.2.0",
repo_name = "org_brotli_fuzz",
)
bazel_dep(name = "rules_fuzzing", version = "0.5.2")
bazel_dep(name = "brotli", version = "1.2.0", repo_name = "org_brotli")
local_path_override(
module_name = "brotli",
path = "../..",
)

323
c/fuzz/MODULE.bazel.lock generated
View File

@@ -1,323 +0,0 @@
{
"lockFileVersion": 16,
"registryFileHashes": {
"https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497",
"https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2",
"https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589",
"https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0",
"https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb",
"https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16",
"https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915",
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed",
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da",
"https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
"https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8",
"https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d",
"https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d",
"https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
"https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/source.json": "3e8379efaaef53ce35b7b8ba419df829315a880cb0a030e5bb45c96d6d5ecb5f",
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
"https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8",
"https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e",
"https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686",
"https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a",
"https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5",
"https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d",
"https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651",
"https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138",
"https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917",
"https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d",
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b",
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953",
"https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
"https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
"https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
"https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6",
"https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4",
"https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f",
"https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075",
"https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d",
"https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
"https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5",
"https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5",
"https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
"https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
"https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
"https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
"https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d",
"https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
"https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c",
"https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d",
"https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df",
"https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e",
"https://bcr.bazel.build/modules/protobuf/29.0/source.json": "b857f93c796750eef95f0d61ee378f3420d00ee1dd38627b27193aa482f4f981",
"https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022",
"https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206",
"https://bcr.bazel.build/modules/re2/2023-09-01/source.json": "e044ce89c2883cd957a2969a43e79f7752f9656f6b20050b62f90ede21ec6eb4",
"https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8",
"https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e",
"https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647",
"https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002",
"https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191",
"https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac",
"https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc",
"https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87",
"https://bcr.bazel.build/modules/rules_cc/0.0.16/source.json": "227e83737046aa4f50015da48e98e0d8ab42fd0ec74d8d653b6cc9f9a357f200",
"https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
"https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f",
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
"https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
"https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
"https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
"https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
"https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39",
"https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6",
"https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31",
"https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a",
"https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6",
"https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
"https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2",
"https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
"https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2",
"https://bcr.bazel.build/modules/rules_java/8.6.1/source.json": "f18d9ad3c4c54945bf422ad584fa6c5ca5b3116ff55a5b1bc77e5c1210be5960",
"https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
"https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909",
"https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036",
"https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d",
"https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4",
"https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0",
"https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197",
"https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59",
"https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3",
"https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5",
"https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0",
"https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d",
"https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c",
"https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb",
"https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc",
"https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff",
"https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a",
"https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06",
"https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7",
"https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73",
"https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2",
"https://bcr.bazel.build/modules/rules_proto/7.0.2/source.json": "1e5e7260ae32ef4f2b52fd1d0de8d03b606a44c91b694d2f1afb1d3b28a48ce1",
"https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f",
"https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300",
"https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382",
"https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed",
"https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58",
"https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c",
"https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7",
"https://bcr.bazel.build/modules/rules_python/0.40.0/source.json": "939d4bd2e3110f27bfb360292986bb79fd8dcefb874358ccd6cdaa7bda029320",
"https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
"https://bcr.bazel.build/modules/rules_shell/0.2.0/source.json": "7f27af3c28037d9701487c4744b5448d26537cc66cdef0d8df7ae85411f8de95",
"https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
"https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
"https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef",
"https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c",
"https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7",
"https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01",
"https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
"https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d",
"https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198"
},
"selectedYankedVersions": {},
"moduleExtensions": {
"@@platforms//host:extension.bzl%host_platform": {
"general": {
"bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=",
"usagesDigest": "SeQiIN/f8/Qt9vYQk7qcXp4I4wJeEC0RnQDiaaJ4tb8=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"host_platform": {
"repoRuleId": "@@platforms//host:extension.bzl%host_platform_repo",
"attributes": {}
}
},
"recordedRepoMappingEntries": []
}
},
"@@rules_fuzzing+//fuzzing/private:extensions.bzl%non_module_dependencies": {
"general": {
"bzlTransitiveDigest": "mGiTB79hRNjmeDTQdzkpCHyzXhErMbufeAmySBt7s5s=",
"usagesDigest": "wy6ISK6UOcBEjj/mvJ/S3WeXoO67X+1llb9yPyFtPgc=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"platforms": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"urls": [
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
"https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz"
],
"sha256": "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74"
}
},
"rules_python": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"sha256": "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8",
"strip_prefix": "rules_python-0.28.0",
"url": "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz"
}
},
"bazel_skylib": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"sha256": "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94",
"urls": [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz"
]
}
},
"com_google_absl": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"urls": [
"https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.1.zip"
],
"strip_prefix": "abseil-cpp-20240116.1",
"integrity": "sha256-7capMWOvWyoYbUaHF/b+I2U6XLMaHmky8KugWvfXYuk="
}
},
"rules_fuzzing_oss_fuzz": {
"repoRuleId": "@@rules_fuzzing+//fuzzing/private/oss_fuzz:repository.bzl%oss_fuzz_repository",
"attributes": {}
},
"honggfuzz": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"build_file": "@@rules_fuzzing+//:honggfuzz.BUILD",
"sha256": "6b18ba13bc1f36b7b950c72d80f19ea67fbadc0ac0bb297ec89ad91f2eaa423e",
"url": "https://github.com/google/honggfuzz/archive/2.5.zip",
"strip_prefix": "honggfuzz-2.5"
}
},
"rules_fuzzing_jazzer": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar",
"attributes": {
"sha256": "ee6feb569d88962d59cb59e8a31eb9d007c82683f3ebc64955fd5b96f277eec2",
"url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer/0.20.1/jazzer-0.20.1.jar"
}
},
"rules_fuzzing_jazzer_api": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar",
"attributes": {
"sha256": "f5a60242bc408f7fa20fccf10d6c5c5ea1fcb3c6f44642fec5af88373ae7aa1b",
"url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer-api/0.20.1/jazzer-api-0.20.1.jar"
}
}
},
"recordedRepoMappingEntries": [
[
"rules_fuzzing+",
"bazel_tools",
"bazel_tools"
]
]
}
},
"@@rules_java+//java:rules_java_deps.bzl%compatibility_proxy": {
"general": {
"bzlTransitiveDigest": "84xJEZ1jnXXwo8BXMprvBm++rRt4jsTu9liBxz0ivps=",
"usagesDigest": "jTQDdLDxsS43zuRmg1faAjIEPWdLAbDAowI1pInQSoo=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"compatibility_proxy": {
"repoRuleId": "@@rules_java+//java:rules_java_deps.bzl%_compatibility_proxy_repo_rule",
"attributes": {}
}
},
"recordedRepoMappingEntries": [
[
"rules_java+",
"bazel_tools",
"bazel_tools"
]
]
}
},
"@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": {
"general": {
"bzlTransitiveDigest": "sFhcgPbDQehmbD1EOXzX4H1q/CD5df8zwG4kp4jbvr8=",
"usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"com_github_jetbrains_kotlin_git": {
"repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository",
"attributes": {
"urls": [
"https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip"
],
"sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88"
}
},
"com_github_jetbrains_kotlin": {
"repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository",
"attributes": {
"git_repository_name": "com_github_jetbrains_kotlin_git",
"compiler_version": "1.9.23"
}
},
"com_github_google_ksp": {
"repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository",
"attributes": {
"urls": [
"https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip"
],
"sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d",
"strip_version": "1.9.23-1.0.20"
}
},
"com_github_pinterest_ktlint": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file",
"attributes": {
"sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985",
"urls": [
"https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint"
],
"executable": true
}
},
"rules_android": {
"repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive",
"attributes": {
"sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806",
"strip_prefix": "rules_android-0.1.1",
"urls": [
"https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip"
]
}
}
},
"recordedRepoMappingEntries": [
[
"rules_kotlin+",
"bazel_tools",
"bazel_tools"
]
]
}
}
}
}

View File

@@ -1,63 +0,0 @@
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <brotli/decode.h>
// Entry point for LibFuzzer.
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
size_t addend = 0;
if (size > 0)
addend = data[size - 1] & 7;
const uint8_t* next_in = data;
const int kBufferSize = 1024;
uint8_t* buffer = (uint8_t*) malloc(kBufferSize);
if (!buffer) {
// OOM is out-of-scope here.
return 0;
}
/* The biggest "magic number" in brotli is 16MiB - 16, so no need to check
the cases with much longer output. */
const size_t total_out_limit = (addend == 0) ? (1 << 26) : (1 << 24);
size_t total_out = 0;
BrotliDecoderState* state = BrotliDecoderCreateInstance(0, 0, 0);
if (!state) {
// OOM is out-of-scope here.
free(buffer);
return 0;
}
if (addend == 0)
addend = size;
/* Test both fast (addend == size) and slow (addend <= 7) decoding paths. */
for (size_t i = 0; i < size;) {
size_t next_i = i + addend;
if (next_i > size)
next_i = size;
size_t avail_in = next_i - i;
i = next_i;
BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
size_t avail_out = kBufferSize;
uint8_t* next_out = buffer;
result = BrotliDecoderDecompressStream(
state, &avail_in, &next_in, &avail_out, &next_out, &total_out);
if (total_out > total_out_limit)
break;
}
if (total_out > total_out_limit)
break;
if (result != BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT)
break;
}
BrotliDecoderDestroyInstance(state);
free(buffer);
return 0;
}

View File

@@ -1,44 +0,0 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Simple runner for decode_fuzzer.cc */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
void LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
int main(int argc, char* *argv) {
if (argc != 2) {
fprintf(stderr, "Exactly one argument is expected.\n");
exit(EXIT_FAILURE);
}
FILE* f = fopen(argv[1], "r");
if (!f) {
fprintf(stderr, "Failed to open input file.");
exit(EXIT_FAILURE);
}
size_t max_len = 1 << 20;
unsigned char* tmp = (unsigned char*)malloc(max_len);
size_t len = fread(tmp, 1, max_len, f);
if (ferror(f)) {
fclose(f);
fprintf(stderr, "Failed read input file.");
exit(EXIT_FAILURE);
}
/* Make data after the end "inaccessible". */
unsigned char* data = (unsigned char*)malloc(len);
memcpy(data, tmp, len);
free(tmp);
LLVMFuzzerTestOneInput(data, len);
free(data);
exit(EXIT_SUCCESS);
}

Some files were not shown because too many files have changed in this diff Show More