diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 38d5c91..62c365c 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -72,6 +72,54 @@ else() message(STATUS "Brotli: Failed to build from source") endif() +# Download and build OpenZL from git using FetchContent +# OpenZL requires CMake >= 3.20.2 and a C++ compiler (its project() enables CXX) +set(OPENZL_AVAILABLE FALSE) +include(CheckLanguage) +check_language(CXX) + +if(CMAKE_VERSION VERSION_LESS 3.20.2) + message(STATUS "OpenZL: Skipped (requires CMake >= 3.20.2)") +elseif(NOT CMAKE_CXX_COMPILER) + message(STATUS "OpenZL: Skipped (requires a C++ compiler)") +else() + message(STATUS "Fetching OpenZL from GitHub...") + + FetchContent_Declare( + openzl + GIT_REPOSITORY https://github.com/facebook/openzl.git + GIT_TAG v0.3.0 + GIT_SHALLOW TRUE + GIT_SUBMODULES deps/zstd deps/lz4 + ) + + # Configure OpenZL options - build the static C library only + set(OPENZL_BUILD_SHARED_LIBS OFF CACHE BOOL "Build OpenZL as shared library" FORCE) + set(OPENZL_BUILD_TESTS OFF CACHE BOOL "Build OpenZL tests" FORCE) + set(OPENZL_BUILD_BENCHMARKS OFF CACHE BOOL "Build OpenZL benchmarks" FORCE) + set(OPENZL_BUILD_CPP OFF CACHE BOOL "Build OpenZL C++ bindings" FORCE) + set(OPENZL_BUILD_TOOLS OFF CACHE BOOL "Build OpenZL tools" FORCE) + set(OPENZL_BUILD_CLI OFF CACHE BOOL "Build OpenZL CLI" FORCE) + set(OPENZL_BUILD_EXAMPLES OFF CACHE BOOL "Build OpenZL examples" FORCE) + set(OPENZL_BUILD_CUSTOM_PARSERS OFF CACHE BOOL "Build OpenZL custom parsers" FORCE) + set(OPENZL_BUILD_PYTHON_EXT OFF CACHE BOOL "Build OpenZL Python extension" FORCE) + set(OPENZL_INSTALL OFF CACHE BOOL "Install OpenZL" FORCE) + set(OPENZL_CPP_INSTALL OFF CACHE BOOL "Install OpenZL C++ bindings" FORCE) + # OpenZL bundles its own zstd, build it static only + set(ZSTD_BUILD_SHARED OFF CACHE BOOL "Build zstd shared library" FORCE) + set(ZSTD_BUILD_STATIC ON CACHE BOOL "Build zstd static library" FORCE) + + # Make OpenZL available + FetchContent_MakeAvailable(openzl) + + if(TARGET openzl) + set(OPENZL_AVAILABLE TRUE) + message(STATUS "OpenZL: Built from source (FetchContent)") + else() + message(STATUS "OpenZL: Failed to build from source") + endif() +endif() + # Benchmark executable add_executable(aarubenchmark @@ -113,6 +161,13 @@ if(BROTLI_AVAILABLE) target_include_directories(aarubenchmark PRIVATE ${BROTLI_SOURCE_DIR}/c/include) endif() +# Add OpenZL compile definition if available (includes come from openzl target) +if(OPENZL_AVAILABLE) + target_compile_definitions(aarubenchmark PRIVATE HAVE_OPENZL=1) + # OpenZL public headers require C11 + set_target_properties(aarubenchmark PROPERTIES C_STANDARD 11) +endif() + # Link to the main library and compression libraries target_link_libraries(aarubenchmark PRIVATE @@ -145,6 +200,11 @@ if(BROTLI_AVAILABLE) target_link_libraries(aarubenchmark PRIVATE brotlienc brotlidec brotlicommon) endif() +# Link OpenZL if available +if(OPENZL_AVAILABLE) + target_link_libraries(aarubenchmark PRIVATE openzl) +endif() + # On Linux, enable GNU/POSIX feature test macros if(UNIX AND NOT APPLE) @@ -172,4 +232,9 @@ if(BROTLI_AVAILABLE) else() message(STATUS " - Brotli support: NO (will be disabled)") endif() +if(OPENZL_AVAILABLE) + message(STATUS " - OpenZL support: YES (downloaded from git)") +else() + message(STATUS " - OpenZL support: NO (will be disabled)") +endif() diff --git a/benchmark/README.md b/benchmark/README.md index 53741a6..2858b70 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -5,7 +5,7 @@ This tool benchmarks different compression algorithms on Aaru format images with ## Purpose The benchmark tool helps determine the most effective compression algorithm for specific image types by: -- Testing multiple compression algorithms (LZMA, Bzip3, Zstd) +- Testing multiple compression algorithms (LZMA, Bzip3, Brotli, OpenZL, Zstd) - Measuring compression ratios and processing times - Providing detailed performance metrics @@ -74,6 +74,8 @@ Fastest: Zstd Algorithms without available libraries will be skipped automatically. +OpenZL is fetched and built from source, which requires CMake >= 3.20.2 and a C++ compiler; it is disabled otherwise. + ## Building The benchmark tool is built automatically when building the main project: diff --git a/benchmark/benchmark.c b/benchmark/benchmark.c index 7eabdd2..f5d6908 100644 --- a/benchmark/benchmark.c +++ b/benchmark/benchmark.c @@ -184,6 +184,7 @@ int main(int argc, char *argv[]) printf(" %s•%s LZMA %s(high compression, slow)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); printf(" %s•%s Bzip3 %s(high compression, medium speed)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); printf(" %s•%s Brotli %s(good compression, medium speed)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); + printf(" %s•%s OpenZL %s(format-aware, generic graph)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); printf(" %s•%s Zstd %s(good compression, fast)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); printf(" %s•%s Zstd+Dict %s(better compression with trained dictionary)%s\n\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET)); return 1; @@ -400,11 +401,11 @@ int main(int argc, char *argv[]) // ===== END DICTIONARY TRAINING ===== // Test each compression algorithm - const compression_algorithm algorithms[] = {COMP_LZMA, COMP_BZIP3, COMP_BROTLI, COMP_ZSTD, COMP_ZSTD}; - const char *algorithm_names[] = {"LZMA", "Bzip3", "Brotli", "Zstd (no dict)", "Zstd (with dict)"}; - const size_t algorithm_count = sizeof(algorithms) / sizeof(algorithms[0]); + const compression_algorithm algorithms[] = {COMP_LZMA, COMP_BZIP3, COMP_BROTLI, COMP_OPENZL, COMP_ZSTD, COMP_ZSTD}; + const char *algorithm_names[] = {"LZMA", "Bzip3", "Brotli", "OpenZL", "Zstd (no dict)", "Zstd (with dict)"}; + const size_t algorithm_count = sizeof(algorithms) / sizeof(algorithms[0]); - benchmark_result results[5]; + benchmark_result results[sizeof(algorithms) / sizeof(algorithms[0])]; print_section_header("Compression Benchmarks"); @@ -420,8 +421,8 @@ int main(int argc, char *argv[]) progress_state progress = {0, info.block_count, {0}}; snprintf(progress.label, sizeof(progress.label), "Compressing"); - // Pass dictionary only for fifth run (Zstd with dict) - const zstd_dict_context *use_dict = (i == 4) ? dict_ctx : NULL; + // Pass dictionary only for last run (Zstd with dict) + const zstd_dict_context *use_dict = (i == algorithm_count - 1) ? dict_ctx : NULL; if(benchmark_compression(input_path, output_path, algorithms[i], &info, &results[i], &progress, print_progress, use_dict) != 0) diff --git a/benchmark/benchmark.h b/benchmark/benchmark.h index a7a3e4c..af4858f 100644 --- a/benchmark/benchmark.h +++ b/benchmark/benchmark.h @@ -37,7 +37,8 @@ typedef enum COMP_LZMA = 0, COMP_BZIP3 = 1, COMP_ZSTD = 2, - COMP_BROTLI = 3 + COMP_BROTLI = 3, + COMP_OPENZL = 4 } compression_algorithm; // Image information structure diff --git a/benchmark/compression.c b/benchmark/compression.c index d25b8b6..e5f0826 100644 --- a/benchmark/compression.c +++ b/benchmark/compression.c @@ -37,6 +37,10 @@ #include #endif +#ifdef HAVE_OPENZL +#include +#endif + // LZMA compression from library extern int32_t aaruf_lzma_encode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, @@ -211,6 +215,46 @@ static int compress_brotli(const uint8_t *input, const size_t input_size, uint8_ #endif } +#ifdef HAVE_OPENZL +// OpenZL graph function: sets global parameters and selects the generic graph +// Compression level 19 to match the Zstd run (passed through to the zstd backend) +static ZL_GraphID openzl_graph(ZL_Compressor *compressor) +{ + ZL_Compressor_setParameter(compressor, ZL_CParam_formatVersion, ZL_MAX_FORMAT_VERSION); + ZL_Compressor_setParameter(compressor, ZL_CParam_compressionLevel, 19); + return ZL_GRAPH_COMPRESS_GENERIC; +} +#endif + +// Compress data using OpenZL +static int compress_openzl(const uint8_t *input, const size_t input_size, uint8_t **output, size_t *output_size) +{ +#ifdef HAVE_OPENZL + // Calculate max output size + const size_t max_output_size = ZL_compressBound(input_size); + *output = malloc(max_output_size); + if(*output == NULL) return -1; + + const ZL_Report result = ZL_compress_usingGraphFn(*output, max_output_size, input, input_size, openzl_graph); + + if(ZL_isError(result)) + { + free(*output); + *output = NULL; + return -1; + } + + *output_size = ZL_validResult(result); + return 0; +#else + // OpenZL not available + (void)input; + (void)input_size; + *output = NULL; + *output_size = 0; + return -1; +#endif +} // Main compression function int compress_data(const compression_algorithm algorithm, const uint8_t *input, const size_t input_size, @@ -226,6 +270,8 @@ int compress_data(const compression_algorithm algorithm, const uint8_t *input, c return compress_zstd(input, input_size, output, output_size); case COMP_BROTLI: return compress_brotli(input, input_size, output, output_size); + case COMP_OPENZL: + return compress_openzl(input, input_size, output, output_size); default: return -1; } @@ -244,6 +290,8 @@ int get_compression_type(const compression_algorithm algorithm) return 101; // Custom identifier for zstd case COMP_BROTLI: return 102; // Custom identifier for brotli + case COMP_OPENZL: + return 103; // Custom identifier for openzl default: return 0; // None }