mirror of
https://github.com/aaru-dps/libaaruformat.git
synced 2026-07-08 18:06:18 +00:00
Add Zstandard compression support
Implements zstd as an alternative to LZMA for data block and subchannel compression using the allocated compression IDs 4 (kCompressionZstd) and 5 (kCompressionZstdCst). Adds zstd v1.5.7 as a bundled submodule with a static library build including x86_64 assembly fast path for decompression. Write path selects zstd or LZMA based on the zstd option. Subchannel blocks use zstd+CST instead of LZMA+CST when enabled. LZMA properties header is only written for LZMA-based compression types, preventing format corruption in zstd-compressed images. Activated via options string: "zstd=true;zstd_level=19". Default remains LZMA for backwards compatibility. Tested: SHA-256 verified lossless roundtrips across 9 disc systems (Dreamcast, Saturn, Mega CD, PC Engine CD, Neo Geo CD, PS1, PS2 CD, PS2 DVD) and PS1 SBI subchannel preservation with zstd+CST.
This commit is contained in:
3
.gitmodules
vendored
3
.gitmodules
vendored
@@ -16,3 +16,6 @@
|
||||
[submodule "3rdparty/BLAKE3"]
|
||||
path = 3rdparty/BLAKE3
|
||||
url = https://github.com/BLAKE3-team/BLAKE3
|
||||
[submodule "3rdparty/zstd"]
|
||||
path = 3rdparty/zstd
|
||||
url = https://github.com/facebook/zstd.git
|
||||
|
||||
1
3rdparty/zstd
vendored
Submodule
1
3rdparty/zstd
vendored
Submodule
Submodule 3rdparty/zstd added at f8745da6ff
60
3rdparty/zstd.cmake
vendored
Normal file
60
3rdparty/zstd.cmake
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
# Zstandard compression library (https://github.com/facebook/zstd)
|
||||
# BSD/GPLv2 dual license
|
||||
|
||||
set(ZSTD_LIB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/zstd/lib")
|
||||
|
||||
if(NOT EXISTS "${ZSTD_LIB_DIR}/zstd.h")
|
||||
message(FATAL_ERROR "Zstandard submodule not found. Run: git submodule update --init 3rdparty/zstd")
|
||||
endif()
|
||||
|
||||
message(STATUS "Zstandard: Building static library")
|
||||
|
||||
add_library(zstd_static STATIC
|
||||
${ZSTD_LIB_DIR}/common/debug.c
|
||||
${ZSTD_LIB_DIR}/common/entropy_common.c
|
||||
${ZSTD_LIB_DIR}/common/error_private.c
|
||||
${ZSTD_LIB_DIR}/common/fse_decompress.c
|
||||
${ZSTD_LIB_DIR}/common/pool.c
|
||||
${ZSTD_LIB_DIR}/common/threading.c
|
||||
${ZSTD_LIB_DIR}/common/xxhash.c
|
||||
${ZSTD_LIB_DIR}/common/zstd_common.c
|
||||
${ZSTD_LIB_DIR}/compress/fse_compress.c
|
||||
${ZSTD_LIB_DIR}/compress/hist.c
|
||||
${ZSTD_LIB_DIR}/compress/huf_compress.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_compress.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_compress_literals.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_compress_sequences.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_compress_superblock.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_double_fast.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_fast.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_lazy.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_ldm.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_opt.c
|
||||
${ZSTD_LIB_DIR}/compress/zstd_preSplit.c
|
||||
${ZSTD_LIB_DIR}/compress/zstdmt_compress.c
|
||||
${ZSTD_LIB_DIR}/decompress/huf_decompress.c
|
||||
${ZSTD_LIB_DIR}/decompress/zstd_ddict.c
|
||||
${ZSTD_LIB_DIR}/decompress/zstd_decompress.c
|
||||
${ZSTD_LIB_DIR}/decompress/zstd_decompress_block.c
|
||||
)
|
||||
|
||||
# x86_64 assembly fast path for Huffman decompression
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
|
||||
enable_language(ASM)
|
||||
target_sources(zstd_static PRIVATE ${ZSTD_LIB_DIR}/decompress/huf_decompress_amd64.S)
|
||||
endif()
|
||||
|
||||
target_include_directories(zstd_static PUBLIC ${ZSTD_LIB_DIR})
|
||||
set_property(TARGET zstd_static PROPERTY POSITION_INDEPENDENT_CODE TRUE)
|
||||
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||
if(MSVC)
|
||||
target_compile_options(zstd_static PRIVATE /O2)
|
||||
else()
|
||||
target_compile_options(zstd_static PRIVATE -O3)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Link to aaruformat
|
||||
target_link_libraries(aaruformat zstd_static)
|
||||
target_include_directories(aaruformat PRIVATE ${ZSTD_LIB_DIR})
|
||||
@@ -268,7 +268,8 @@ add_library(aaruformat
|
||||
src/ngcw/ngcw_junk.c
|
||||
src/ngcw/ngcw_junk.h
|
||||
src/ngcw/wii_crypto.c
|
||||
src/ngcw/wii_crypto.h)
|
||||
src/ngcw/wii_crypto.h
|
||||
src/compression/zstd.c)
|
||||
|
||||
# Set up include directories for the target
|
||||
target_include_directories(aaruformat
|
||||
@@ -287,6 +288,7 @@ include(3rdparty/flac.cmake)
|
||||
include(3rdparty/lzma.cmake)
|
||||
include(3rdparty/xxhash.cmake)
|
||||
include(3rdparty/blake3.cmake)
|
||||
include(3rdparty/zstd.cmake)
|
||||
|
||||
if(TARGET blake3)
|
||||
target_link_libraries(aaruformat blake3)
|
||||
|
||||
@@ -301,6 +301,8 @@ typedef struct aaruformat_context
|
||||
uint32_t lzma_dict_size; ///< LZMA dictionary size (writing path).
|
||||
bool deduplicate; ///< Storage deduplication active (duplicates coalesce).
|
||||
bool compression_enabled; ///< True if block compression enabled (writing path).
|
||||
bool use_zstd; ///< Use Zstandard instead of LZMA for data blocks.
|
||||
int zstd_level; ///< Zstandard compression level (writing path, default 19).
|
||||
|
||||
/* Tape-specific structures */
|
||||
tapeFileHashEntry *tape_files; ///< Hash table root for tape files
|
||||
|
||||
@@ -254,6 +254,11 @@ AARU_EXPORT int32_t AARU_CALL aaruf_lzma_encode_buffer(uint8_t *dst_buffer, size
|
||||
int32_t level, uint32_t dict_size, int32_t lc, int32_t lp,
|
||||
int32_t pb, int32_t fb, int32_t num_threads);
|
||||
|
||||
AARU_EXPORT size_t AARU_CALL aaruf_zstd_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer,
|
||||
size_t src_size);
|
||||
AARU_EXPORT size_t AARU_CALL aaruf_zstd_encode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer,
|
||||
size_t src_size, int level);
|
||||
|
||||
AARU_EXPORT void AARU_CALL aaruf_md5_init(md5_ctx *ctx);
|
||||
AARU_EXPORT void AARU_CALL aaruf_md5_update(md5_ctx *ctx, const void *data, unsigned long size);
|
||||
AARU_EXPORT void AARU_CALL aaruf_md5_final(md5_ctx *ctx, unsigned char *result);
|
||||
|
||||
@@ -34,8 +34,8 @@ typedef enum
|
||||
kCompressionLzma = 1, ///< LZMA compression.
|
||||
kCompressionFlac = 2, ///< FLAC compression.
|
||||
kCompressionLzmaCst = 3, ///< LZMA applied to Claunia Subchannel Transform processed data.
|
||||
kCompressionZstd = 4, ///< Zstandard compression (reserved for future implementation).
|
||||
kCompressionZstdCst = 5 ///< Zstandard applied to Claunia Subchannel Transform processed data (reserved).
|
||||
kCompressionZstd = 4, ///< Zstandard compression.
|
||||
kCompressionZstdCst = 5 ///< Zstandard applied to Claunia Subchannel Transform processed data.
|
||||
} CompressionType;
|
||||
|
||||
/**
|
||||
|
||||
@@ -227,6 +227,8 @@ typedef struct
|
||||
bool sha256; ///< Generate SHA-256 checksum (ChecksumAlgorithm::Sha256) when finalizing image.
|
||||
bool blake3; ///< Generate BLAKE3 checksum if supported (not stored if algorithm unavailable).
|
||||
bool spamsum; ///< Generate SpamSum fuzzy hash (ChecksumAlgorithm::SpamSum) if enabled.
|
||||
bool zstd; ///< Use Zstandard instead of LZMA for data blocks. Default: false.
|
||||
int zstd_level; ///< Zstandard compression level (1-22). Default: 19.
|
||||
} aaru_options;
|
||||
|
||||
#endif // LIBAARUFORMAT_OPTIONS_H
|
||||
|
||||
@@ -281,6 +281,85 @@ int32_t process_data_block(aaruformat_context *ctx, IndexEntry *entry)
|
||||
|
||||
free(cmp_data);
|
||||
}
|
||||
else if(block_header.compression == kCompressionZstd || block_header.compression == kCompressionZstdCst)
|
||||
{
|
||||
if(block_header.compression == kCompressionZstdCst && block_header.type != kDataTypeCdSubchannel)
|
||||
{
|
||||
TRACE("Invalid compression type %u for block with data type %u, continuing...", block_header.compression,
|
||||
block_header.type);
|
||||
TRACE("Exiting process_data_block() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
|
||||
cmp_data = (block_header.cmpLength == 0) ? NULL : (uint8_t *)malloc(block_header.cmpLength);
|
||||
if(block_header.cmpLength != 0 && cmp_data == NULL)
|
||||
{
|
||||
TRACE("Cannot allocate memory for compressed block, continuing...");
|
||||
TRACE("Exiting process_data_block() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
|
||||
if(block_header.length != 0)
|
||||
{
|
||||
data = (uint8_t *)malloc(block_header.length);
|
||||
if(data == NULL)
|
||||
{
|
||||
TRACE("Cannot allocate memory for block, continuing...");
|
||||
free(cmp_data);
|
||||
TRACE("Exiting process_data_block() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
}
|
||||
else
|
||||
data = NULL;
|
||||
|
||||
if(block_header.cmpLength != 0)
|
||||
{
|
||||
read_bytes = fread(cmp_data, 1, block_header.cmpLength, ctx->imageStream);
|
||||
if(read_bytes != block_header.cmpLength)
|
||||
{
|
||||
TRACE("Could not read compressed block, continuing...");
|
||||
free(cmp_data);
|
||||
free(data);
|
||||
TRACE("Exiting process_data_block() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
}
|
||||
|
||||
if(block_header.length != 0)
|
||||
{
|
||||
size_t decoded = aaruf_zstd_decode_buffer(data, block_header.length, cmp_data, block_header.cmpLength);
|
||||
|
||||
if(decoded != block_header.length)
|
||||
{
|
||||
TRACE("Error decompressing zstd block, expected %" PRIu32 " bytes but got %zu bytes, continuing...",
|
||||
block_header.length, decoded);
|
||||
free(cmp_data);
|
||||
free(data);
|
||||
TRACE("Exiting process_data_block() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
free(cmp_data);
|
||||
|
||||
if(block_header.compression == kCompressionZstdCst && block_header.length != 0)
|
||||
{
|
||||
cst_data = (uint8_t *)malloc(block_header.length);
|
||||
if(cst_data == NULL)
|
||||
{
|
||||
TRACE("Cannot allocate memory for CST untransform, continuing...");
|
||||
free(data);
|
||||
TRACE("Exiting process_data_block() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
|
||||
aaruf_cst_untransform(data, cst_data, block_header.length);
|
||||
free(data);
|
||||
data = cst_data;
|
||||
cst_data = NULL;
|
||||
}
|
||||
}
|
||||
else if(block_header.compression == kCompressionNone)
|
||||
{
|
||||
if(block_header.length != 0)
|
||||
|
||||
61
src/close.c
61
src/close.c
@@ -1698,23 +1698,35 @@ static void write_sector_subchannel(aaruformat_context *ctx)
|
||||
|
||||
if(dst_buffer == NULL)
|
||||
{
|
||||
TRACE("Failed to allocate memory for LZMA output");
|
||||
TRACE("Failed to allocate memory for compressed output");
|
||||
free(cst_buffer);
|
||||
return;
|
||||
}
|
||||
|
||||
aaruf_cst_transform(ctx->sector_subchannel, cst_buffer, subchannel_block.length);
|
||||
size_t dst_size = subchannel_block.length;
|
||||
size_t props_size = LZMA_PROPERTIES_LENGTH;
|
||||
|
||||
aaruf_lzma_encode_buffer(dst_buffer, &dst_size, cst_buffer, subchannel_block.length, lzma_properties,
|
||||
&props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
|
||||
size_t dst_size;
|
||||
|
||||
if(ctx->use_zstd)
|
||||
{
|
||||
dst_size = aaruf_zstd_encode_buffer(dst_buffer, subchannel_block.length, cst_buffer,
|
||||
subchannel_block.length, ctx->zstd_level);
|
||||
if(dst_size == 0) dst_size = subchannel_block.length; /* compression failed, fall through to none */
|
||||
}
|
||||
else
|
||||
{
|
||||
dst_size = subchannel_block.length;
|
||||
size_t props_size = LZMA_PROPERTIES_LENGTH;
|
||||
|
||||
aaruf_lzma_encode_buffer(dst_buffer, &dst_size, cst_buffer, subchannel_block.length, lzma_properties,
|
||||
&props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
|
||||
}
|
||||
|
||||
free(cst_buffer);
|
||||
|
||||
if(dst_size < subchannel_block.length)
|
||||
{
|
||||
subchannel_block.compression = kCompressionLzmaCst;
|
||||
subchannel_block.compression = ctx->use_zstd ? kCompressionZstdCst : kCompressionLzmaCst;
|
||||
subchannel_block.cmpLength = (uint32_t)dst_size;
|
||||
buffer = dst_buffer;
|
||||
owns_buffer = true;
|
||||
@@ -1749,28 +1761,39 @@ static void write_sector_subchannel(aaruformat_context *ctx)
|
||||
TRACE("Incorrect media type, not writing sector subchannel block");
|
||||
return; // Incorrect media type
|
||||
}
|
||||
subchannel_block.cmpLength = subchannel_block.length;
|
||||
subchannel_block.compression = kCompressionLzma;
|
||||
subchannel_block.cmpLength = subchannel_block.length;
|
||||
|
||||
uint8_t *dst_buffer = malloc(subchannel_block.length);
|
||||
|
||||
if(dst_buffer == NULL)
|
||||
{
|
||||
TRACE("Failed to allocate memory for LZMA output");
|
||||
TRACE("Failed to allocate memory for compressed output");
|
||||
return;
|
||||
}
|
||||
|
||||
size_t dst_size = subchannel_block.length;
|
||||
size_t props_size = LZMA_PROPERTIES_LENGTH;
|
||||
size_t dst_size;
|
||||
|
||||
aaruf_lzma_encode_buffer(dst_buffer, &dst_size, ctx->sector_subchannel, subchannel_block.length,
|
||||
lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
|
||||
if(ctx->use_zstd)
|
||||
{
|
||||
dst_size = aaruf_zstd_encode_buffer(dst_buffer, subchannel_block.length, ctx->sector_subchannel,
|
||||
subchannel_block.length, ctx->zstd_level);
|
||||
if(dst_size == 0) dst_size = subchannel_block.length; /* compression failed, fall through to none */
|
||||
}
|
||||
else
|
||||
{
|
||||
dst_size = subchannel_block.length;
|
||||
size_t props_size = LZMA_PROPERTIES_LENGTH;
|
||||
|
||||
aaruf_lzma_encode_buffer(dst_buffer, &dst_size, ctx->sector_subchannel, subchannel_block.length,
|
||||
lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
|
||||
}
|
||||
|
||||
if(dst_size < subchannel_block.length)
|
||||
{
|
||||
subchannel_block.cmpLength = (uint32_t)dst_size;
|
||||
buffer = dst_buffer;
|
||||
owns_buffer = true;
|
||||
subchannel_block.compression = ctx->use_zstd ? kCompressionZstd : kCompressionLzma;
|
||||
subchannel_block.cmpLength = (uint32_t)dst_size;
|
||||
buffer = dst_buffer;
|
||||
owns_buffer = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -1793,12 +1816,14 @@ static void write_sector_subchannel(aaruformat_context *ctx)
|
||||
subchannel_block.cmpCrc64 = aaruf_crc64_data(buffer, subchannel_block.cmpLength);
|
||||
|
||||
const size_t length_to_write = subchannel_block.cmpLength;
|
||||
if(subchannel_block.compression != kCompressionNone) subchannel_block.cmpLength += LZMA_PROPERTIES_LENGTH;
|
||||
const bool has_lzma_props = subchannel_block.compression == kCompressionLzma ||
|
||||
subchannel_block.compression == kCompressionLzmaCst;
|
||||
if(has_lzma_props) subchannel_block.cmpLength += LZMA_PROPERTIES_LENGTH;
|
||||
|
||||
// Write header
|
||||
if(fwrite(&subchannel_block, sizeof(BlockHeader), 1, ctx->imageStream) == 1)
|
||||
{
|
||||
if(subchannel_block.compression != kCompressionNone)
|
||||
if(has_lzma_props)
|
||||
fwrite(lzma_properties, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream);
|
||||
|
||||
// Write data
|
||||
|
||||
62
src/compression/zstd.c
Normal file
62
src/compression/zstd.c
Normal file
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* This file is part of the Aaru Data Preservation Suite.
|
||||
* Copyright (c) 2019-2026 Natalia Portillo.
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as
|
||||
* published by the Free Software Foundation; either version 2.1 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <aaruformat.h>
|
||||
#include <zstd.h>
|
||||
|
||||
/**
|
||||
* @brief Decodes a Zstandard-compressed buffer.
|
||||
*
|
||||
* @param dst_buffer Pointer to the destination buffer.
|
||||
* @param dst_size Size of the destination buffer.
|
||||
* @param src_buffer Pointer to the source (compressed) buffer.
|
||||
* @param src_size Size of the source buffer.
|
||||
* @return Number of decompressed bytes, or 0 on error.
|
||||
*/
|
||||
AARU_EXPORT size_t AARU_CALL aaruf_zstd_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer,
|
||||
size_t src_size)
|
||||
{
|
||||
size_t result = ZSTD_decompress(dst_buffer, dst_size, src_buffer, src_size);
|
||||
|
||||
if(ZSTD_isError(result)) return 0;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Encodes a buffer using Zstandard compression.
|
||||
*
|
||||
* @param dst_buffer Pointer to the destination buffer.
|
||||
* @param dst_size Size of the destination buffer.
|
||||
* @param src_buffer Pointer to the source (uncompressed) buffer.
|
||||
* @param src_size Size of the source buffer.
|
||||
* @param level Compression level (1-22).
|
||||
* @return Number of compressed bytes, or 0 on error.
|
||||
*/
|
||||
AARU_EXPORT size_t AARU_CALL aaruf_zstd_encode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer,
|
||||
size_t src_size, int level)
|
||||
{
|
||||
size_t result = ZSTD_compress(dst_buffer, dst_size, src_buffer, src_size, level);
|
||||
|
||||
if(ZSTD_isError(result)) return 0;
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -526,6 +526,8 @@ AARU_EXPORT void AARU_CALL *aaruf_create(const char *filepath, const uint32_t me
|
||||
ctx->compression_enabled = parsed_options.compress;
|
||||
ctx->lzma_dict_size = parsed_options.dictionary;
|
||||
ctx->deduplicate = parsed_options.deduplicate;
|
||||
ctx->use_zstd = parsed_options.zstd;
|
||||
ctx->zstd_level = parsed_options.zstd_level;
|
||||
if(ctx->deduplicate)
|
||||
ctx->sector_hash_map = create_map(ctx->user_data_ddt_header.blocks * 25 / 100); // 25% of total sectors
|
||||
|
||||
|
||||
@@ -50,14 +50,18 @@ aaru_options parse_options(const char *options, bool *table_shift_found)
|
||||
.sha1 = false,
|
||||
.sha256 = false,
|
||||
.blake3 = false,
|
||||
.spamsum = false};
|
||||
.spamsum = false,
|
||||
.zstd = false,
|
||||
.zstd_level = 19};
|
||||
|
||||
if(options == NULL)
|
||||
{
|
||||
TRACE("Exiting parse_options() = {compress: %d, deduplicate: %d, dictionary: %u, table_shift: %d, "
|
||||
"data_shift: %u, block_alignment: %u, md5: %d, sha1: %d, sha256: %d, blake3: %d, spamsum: %d}",
|
||||
"data_shift: %u, block_alignment: %u, md5: %d, sha1: %d, sha256: %d, blake3: %d, spamsum: %d, "
|
||||
"zstd: %d, zstd_level: %d}",
|
||||
parsed.compress, parsed.deduplicate, parsed.dictionary, parsed.table_shift, parsed.data_shift,
|
||||
parsed.block_alignment, parsed.md5, parsed.sha1, parsed.sha256, parsed.blake3, parsed.spamsum);
|
||||
parsed.block_alignment, parsed.md5, parsed.sha1, parsed.sha256, parsed.blake3, parsed.spamsum,
|
||||
parsed.zstd, parsed.zstd_level);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
@@ -134,13 +138,23 @@ aaru_options parse_options(const char *options, bool *table_shift_found)
|
||||
parsed.blake3 = bval;
|
||||
else if(strncmp(key, "spamsum", 7) == 0)
|
||||
parsed.spamsum = bval;
|
||||
else if(strncmp(key, "zstd_level", 10) == 0)
|
||||
{
|
||||
parsed.zstd_level = (int)strtol(value, NULL, 10);
|
||||
if(parsed.zstd_level < 1) parsed.zstd_level = 1;
|
||||
if(parsed.zstd_level > 22) parsed.zstd_level = 22;
|
||||
}
|
||||
else if(strncmp(key, "zstd", 4) == 0)
|
||||
parsed.zstd = bval;
|
||||
}
|
||||
token = strtok_r(NULL, ";", &saveptr);
|
||||
}
|
||||
|
||||
TRACE("Exiting parse_options() = {compress: %d, deduplicate: %d, dictionary: %u, table_shift: %d, "
|
||||
"data_shift: %u, block_alignment: %u, md5: %d, sha1: %d, sha256: %d, blake3: %d, spamsum: %d}",
|
||||
"data_shift: %u, block_alignment: %u, md5: %d, sha1: %d, sha256: %d, blake3: %d, spamsum: %d, "
|
||||
"zstd: %d, zstd_level: %d}",
|
||||
parsed.compress, parsed.deduplicate, parsed.dictionary, parsed.table_shift, parsed.data_shift,
|
||||
parsed.block_alignment, parsed.md5, parsed.sha1, parsed.sha256, parsed.blake3, parsed.spamsum);
|
||||
parsed.block_alignment, parsed.md5, parsed.sha1, parsed.sha256, parsed.blake3, parsed.spamsum,
|
||||
parsed.zstd, parsed.zstd_level);
|
||||
return parsed;
|
||||
}
|
||||
53
src/read.c
53
src/read.c
@@ -894,6 +894,59 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
|
||||
free(cmp_data);
|
||||
|
||||
break;
|
||||
case kCompressionZstd:
|
||||
if(block_header->cmpLength == 0 || block_header->length == 0)
|
||||
{
|
||||
FATAL("Invalid zstd block lengths (cmpLength=%u, length=%u)", block_header->cmpLength,
|
||||
block_header->length);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
|
||||
TRACE("Allocating memory for block of size %zu bytes", block_header->length);
|
||||
block = (uint8_t *)malloc(block_header->length);
|
||||
if(block == NULL)
|
||||
{
|
||||
FATAL("Not enough memory for block");
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_NOT_ENOUGH_MEMORY");
|
||||
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
|
||||
}
|
||||
|
||||
TRACE("Allocating memory for compressed data of size %zu bytes", block_header->cmpLength);
|
||||
cmp_data = malloc(block_header->cmpLength);
|
||||
if(cmp_data == NULL)
|
||||
{
|
||||
FATAL("Not enough memory for compressed data");
|
||||
free(block);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_NOT_ENOUGH_MEMORY");
|
||||
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
|
||||
}
|
||||
|
||||
fseek(ctx->imageStream, (long)(block_offset + sizeof(BlockHeader)), SEEK_SET);
|
||||
|
||||
read_bytes = fread(cmp_data, 1, block_header->cmpLength, ctx->imageStream);
|
||||
if(read_bytes != block_header->cmpLength)
|
||||
{
|
||||
FATAL("Could not read compressed block");
|
||||
free(cmp_data);
|
||||
free(block);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
|
||||
read_bytes = aaruf_zstd_decode_buffer(block, block_header->length, cmp_data, block_header->cmpLength);
|
||||
if(read_bytes != block_header->length)
|
||||
{
|
||||
FATAL("Error decompressing zstd block, expected %u bytes got %zu", block_header->length, read_bytes);
|
||||
free(cmp_data);
|
||||
free(block);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
|
||||
free(cmp_data);
|
||||
|
||||
break;
|
||||
case kCompressionFlac:
|
||||
TRACE("Allocating memory for compressed data of size %zu bytes", block_header->cmpLength);
|
||||
|
||||
35
src/write.c
35
src/write.c
@@ -371,7 +371,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_write_sector(void *context, uint64_t sector_
|
||||
if(ctx->current_track_type == kTrackTypeAudio)
|
||||
ctx->current_block_header.compression = kCompressionFlac;
|
||||
else
|
||||
ctx->current_block_header.compression = kCompressionLzma;
|
||||
ctx->current_block_header.compression = ctx->use_zstd ? kCompressionZstd : kCompressionLzma;
|
||||
}
|
||||
else
|
||||
ctx->current_block_header.compression = kCompressionNone;
|
||||
@@ -380,7 +380,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_write_sector(void *context, uint64_t sector_
|
||||
{
|
||||
ctx->current_track_type = kTrackTypeData;
|
||||
if(ctx->compression_enabled)
|
||||
ctx->current_block_header.compression = kCompressionLzma;
|
||||
ctx->current_block_header.compression = ctx->use_zstd ? kCompressionZstd : kCompressionLzma;
|
||||
else
|
||||
ctx->current_block_header.compression = kCompressionNone;
|
||||
}
|
||||
@@ -1576,6 +1576,37 @@ int32_t aaruf_close_current_block(aaruformat_context *ctx)
|
||||
free(cmp_buffer);
|
||||
}
|
||||
|
||||
break;
|
||||
case kCompressionZstd:
|
||||
cmp_buffer = malloc(ctx->current_block_header.length * 2);
|
||||
if(cmp_buffer == NULL)
|
||||
{
|
||||
FATAL("Could not allocate buffer for compressed data");
|
||||
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
|
||||
}
|
||||
|
||||
size_t zstd_dst_size =
|
||||
aaruf_zstd_encode_buffer(cmp_buffer, ctx->current_block_header.length * 2, ctx->writing_buffer,
|
||||
ctx->current_block_header.length, ctx->zstd_level);
|
||||
|
||||
if(zstd_dst_size == 0)
|
||||
{
|
||||
ctx->current_block_header.compression = kCompressionNone;
|
||||
free(cmp_buffer);
|
||||
cmp_buffer = NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
ctx->current_block_header.cmpLength = (uint32_t)zstd_dst_size;
|
||||
|
||||
if(ctx->current_block_header.cmpLength >= ctx->current_block_header.length)
|
||||
{
|
||||
ctx->current_block_header.compression = kCompressionNone;
|
||||
free(cmp_buffer);
|
||||
cmp_buffer = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
default:
|
||||
FATAL("Invalid compression type");
|
||||
|
||||
Reference in New Issue
Block a user