Add OpenZL algorithm to benchmark tool.

2026-11-04 03:15:06 +00:00 · 2026-10-03 15:15:17 +01:00
parent 4bfc28e8b5
commit 1fb3bf7c9b
5 changed files with 125 additions and 8 deletions
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -72,6 +72,54 @@ else()
  message(STATUS "Brotli: Failed to build from source")
 endif()

+# Download and build OpenZL from git using FetchContent
+# OpenZL requires CMake >= 3.20.2 and a C++ compiler (its project() enables CXX)
+set(OPENZL_AVAILABLE FALSE)
+include(CheckLanguage)
+check_language(CXX)
+
+if(CMAKE_VERSION VERSION_LESS 3.20.2)
+  message(STATUS "OpenZL: Skipped (requires CMake >= 3.20.2)")
+elseif(NOT CMAKE_CXX_COMPILER)
+  message(STATUS "OpenZL: Skipped (requires a C++ compiler)")
+else()
+  message(STATUS "Fetching OpenZL from GitHub...")
+
+  FetchContent_Declare(
+          openzl
+          GIT_REPOSITORY https://github.com/facebook/openzl.git
+          GIT_TAG v0.3.0
+          GIT_SHALLOW TRUE
+          GIT_SUBMODULES deps/zstd deps/lz4
+  )
+
+  # Configure OpenZL options - build the static C library only
+  set(OPENZL_BUILD_SHARED_LIBS OFF CACHE BOOL "Build OpenZL as shared library" FORCE)
+  set(OPENZL_BUILD_TESTS OFF CACHE BOOL "Build OpenZL tests" FORCE)
+  set(OPENZL_BUILD_BENCHMARKS OFF CACHE BOOL "Build OpenZL benchmarks" FORCE)
+  set(OPENZL_BUILD_CPP OFF CACHE BOOL "Build OpenZL C++ bindings" FORCE)
+  set(OPENZL_BUILD_TOOLS OFF CACHE BOOL "Build OpenZL tools" FORCE)
+  set(OPENZL_BUILD_CLI OFF CACHE BOOL "Build OpenZL CLI" FORCE)
+  set(OPENZL_BUILD_EXAMPLES OFF CACHE BOOL "Build OpenZL examples" FORCE)
+  set(OPENZL_BUILD_CUSTOM_PARSERS OFF CACHE BOOL "Build OpenZL custom parsers" FORCE)
+  set(OPENZL_BUILD_PYTHON_EXT OFF CACHE BOOL "Build OpenZL Python extension" FORCE)
+  set(OPENZL_INSTALL OFF CACHE BOOL "Install OpenZL" FORCE)
+  set(OPENZL_CPP_INSTALL OFF CACHE BOOL "Install OpenZL C++ bindings" FORCE)
+  # OpenZL bundles its own zstd, build it static only
+  set(ZSTD_BUILD_SHARED OFF CACHE BOOL "Build zstd shared library" FORCE)
+  set(ZSTD_BUILD_STATIC ON CACHE BOOL "Build zstd static library" FORCE)
+
+  # Make OpenZL available
+  FetchContent_MakeAvailable(openzl)
+
+  if(TARGET openzl)
+    set(OPENZL_AVAILABLE TRUE)
+    message(STATUS "OpenZL: Built from source (FetchContent)")
+  else()
+    message(STATUS "OpenZL: Failed to build from source")
+  endif()
+endif()
+

 # Benchmark executable
 add_executable(aarubenchmark
@@ -113,6 +161,13 @@ if(BROTLI_AVAILABLE)
  target_include_directories(aarubenchmark PRIVATE ${BROTLI_SOURCE_DIR}/c/include)
 endif()

+# Add OpenZL compile definition if available (includes come from openzl target)
+if(OPENZL_AVAILABLE)
+  target_compile_definitions(aarubenchmark PRIVATE HAVE_OPENZL=1)
+  # OpenZL public headers require C11
+  set_target_properties(aarubenchmark PROPERTIES C_STANDARD 11)
+endif()
+
 # Link to the main library and compression libraries
 target_link_libraries(aarubenchmark
                      PRIVATE
@@ -145,6 +200,11 @@ if(BROTLI_AVAILABLE)
  target_link_libraries(aarubenchmark PRIVATE brotlienc brotlidec brotlicommon)
 endif()

+# Link OpenZL if available
+if(OPENZL_AVAILABLE)
+  target_link_libraries(aarubenchmark PRIVATE openzl)
+endif()
+

 # On Linux, enable GNU/POSIX feature test macros
 if(UNIX AND NOT APPLE)
@@ -172,4 +232,9 @@ if(BROTLI_AVAILABLE)
 else()
  message(STATUS "  - Brotli support: NO (will be disabled)")
 endif()
+if(OPENZL_AVAILABLE)
+  message(STATUS "  - OpenZL support: YES (downloaded from git)")
+else()
+  message(STATUS "  - OpenZL support: NO (will be disabled)")
+endif()

--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -5,7 +5,7 @@ This tool benchmarks different compression algorithms on Aaru format images with
 ## Purpose

 The benchmark tool helps determine the most effective compression algorithm for specific image types by:
- Testing multiple compression algorithms (LZMA, Bzip3, Zstd)
+- Testing multiple compression algorithms (LZMA, Bzip3, Brotli, OpenZL, Zstd)
 - Measuring compression ratios and processing times
 - Providing detailed performance metrics

@@ -74,6 +74,8 @@ Fastest: Zstd

 Algorithms without available libraries will be skipped automatically.

+OpenZL is fetched and built from source, which requires CMake >= 3.20.2 and a C++ compiler; it is disabled otherwise.
+
 ## Building

 The benchmark tool is built automatically when building the main project:
--- a/benchmark/benchmark.c
+++ b/benchmark/benchmark.c
@@ -184,6 +184,7 @@ int main(int argc, char *argv[])
        printf("  %s•%s LZMA      %s(high compression, slow)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
        printf("  %s•%s Bzip3     %s(high compression, medium speed)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
        printf("  %s•%s Brotli    %s(good compression, medium speed)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
+        printf("  %s•%s OpenZL    %s(format-aware, generic graph)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
        printf("  %s•%s Zstd      %s(good compression, fast)%s\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
        printf("  %s•%s Zstd+Dict %s(better compression with trained dictionary)%s\n\n", clr(ANSI_GREEN), clr(ANSI_RESET), clr(ANSI_DIM), clr(ANSI_RESET));
        return 1;
@@ -400,11 +401,11 @@ int main(int argc, char *argv[])
    // ===== END DICTIONARY TRAINING =====

    // Test each compression algorithm
-    const compression_algorithm algorithms[]      = {COMP_LZMA, COMP_BZIP3, COMP_BROTLI, COMP_ZSTD, COMP_ZSTD};
-    const char                 *algorithm_names[] = {"LZMA", "Bzip3", "Brotli", "Zstd (no dict)", "Zstd (with dict)"};
-    const size_t                algorithm_count   = sizeof(algorithms) / sizeof(algorithms[0]);
+    const compression_algorithm algorithms[] = {COMP_LZMA, COMP_BZIP3, COMP_BROTLI, COMP_OPENZL, COMP_ZSTD, COMP_ZSTD};
+    const char *algorithm_names[] = {"LZMA", "Bzip3", "Brotli", "OpenZL", "Zstd (no dict)", "Zstd (with dict)"};
+    const size_t algorithm_count  = sizeof(algorithms) / sizeof(algorithms[0]);

-    benchmark_result results[5];
+    benchmark_result results[sizeof(algorithms) / sizeof(algorithms[0])];

    print_section_header("Compression Benchmarks");

@@ -420,8 +421,8 @@ int main(int argc, char *argv[])
        progress_state progress = {0, info.block_count, {0}};
        snprintf(progress.label, sizeof(progress.label), "Compressing");

-        // Pass dictionary only for fifth run (Zstd with dict)
-        const zstd_dict_context *use_dict = (i == 4) ? dict_ctx : NULL;
+        // Pass dictionary only for last run (Zstd with dict)
+        const zstd_dict_context *use_dict = (i == algorithm_count - 1) ? dict_ctx : NULL;

        if(benchmark_compression(input_path, output_path, algorithms[i], &info, &results[i], &progress, print_progress,
                                 use_dict) != 0)
--- a/benchmark/benchmark.h
+++ b/benchmark/benchmark.h
@@ -37,7 +37,8 @@ typedef enum
    COMP_LZMA   = 0,
    COMP_BZIP3  = 1,
    COMP_ZSTD   = 2,
-    COMP_BROTLI = 3
+    COMP_BROTLI = 3,
+    COMP_OPENZL = 4
 } compression_algorithm;

 // Image information structure
--- a/benchmark/compression.c
+++ b/benchmark/compression.c
@@ -37,6 +37,10 @@
 #include <brotli/decode.h>
 #endif

+#ifdef HAVE_OPENZL
+#include <openzl/openzl.h>
+#endif
+

 // LZMA compression from library
 extern int32_t aaruf_lzma_encode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer,
@@ -211,6 +215,46 @@ static int compress_brotli(const uint8_t *input, const size_t input_size, uint8_
 #endif
 }

+#ifdef HAVE_OPENZL
+// OpenZL graph function: sets global parameters and selects the generic graph
+// Compression level 19 to match the Zstd run (passed through to the zstd backend)
+static ZL_GraphID openzl_graph(ZL_Compressor *compressor)
+{
+    ZL_Compressor_setParameter(compressor, ZL_CParam_formatVersion, ZL_MAX_FORMAT_VERSION);
+    ZL_Compressor_setParameter(compressor, ZL_CParam_compressionLevel, 19);
+    return ZL_GRAPH_COMPRESS_GENERIC;
+}
+#endif
+
+// Compress data using OpenZL
+static int compress_openzl(const uint8_t *input, const size_t input_size, uint8_t **output, size_t *output_size)
+{
+#ifdef HAVE_OPENZL
+    // Calculate max output size
+    const size_t max_output_size = ZL_compressBound(input_size);
+    *output                      = malloc(max_output_size);
+    if(*output == NULL) return -1;
+
+    const ZL_Report result = ZL_compress_usingGraphFn(*output, max_output_size, input, input_size, openzl_graph);
+
+    if(ZL_isError(result))
+    {
+        free(*output);
+        *output = NULL;
+        return -1;
+    }
+
+    *output_size = ZL_validResult(result);
+    return 0;
+#else
+    // OpenZL not available
+    (void)input;
+    (void)input_size;
+    *output      = NULL;
+    *output_size = 0;
+    return -1;
+#endif
+}

 // Main compression function
 int compress_data(const compression_algorithm algorithm, const uint8_t *input, const size_t input_size,
@@ -226,6 +270,8 @@ int compress_data(const compression_algorithm algorithm, const uint8_t *input, c
            return compress_zstd(input, input_size, output, output_size);
        case COMP_BROTLI:
            return compress_brotli(input, input_size, output, output_size);
+        case COMP_OPENZL:
+            return compress_openzl(input, input_size, output, output_size);
        default:
            return -1;
    }
@@ -244,6 +290,8 @@ int get_compression_type(const compression_algorithm algorithm)
            return 101;  // Custom identifier for zstd
        case COMP_BROTLI:
            return 102;  // Custom identifier for brotli
+        case COMP_OPENZL:
+            return 103;  // Custom identifier for openzl
        default:
            return 0;  // None
    }