ddt_v2.c

Go to the documentation of this file.

/*
 * This file is part of the Aaru Data Preservation Suite.
 * Copyright (c) 2019-2026 Natalia Portillo.
 *
 * This library is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of the
 * License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */
 
#include <inttypes.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
 
#include "aaruformat.h"
#include "internal.h"
#include "log.h"

int32_t process_ddt_v2(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
{
    TRACE("Entering process_ddt_v2(%p, %p, %d)", ctx, entry, *found_user_data_ddt);
 
    int        pos        = 0;
    size_t     read_bytes = 0;
    DdtHeader2 ddt_header;
    uint8_t   *cmp_data = NULL;
    uint8_t    lzma_properties[LZMA_PROPERTIES_LENGTH];
    size_t     lzma_size     = 0;
    int        error_no      = 0;
    crc64_ctx *crc64_context = NULL;
    uint64_t   crc64         = 0;
    uint8_t   *buffer        = NULL;
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
 
        TRACE("Exiting process_ddt_v2() = AARUF_ERROR_NOT_AARUFORMAT");
        return AARUF_ERROR_NOT_AARUFORMAT;
    }
 
    // Seek to block
    pos = fseek(ctx->imageStream, entry->offset, SEEK_SET);
    if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
    {
        FATAL("Could not seek to %" PRIu64 " as indicated by index entry...", entry->offset);
 
        TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
        return AARUF_ERROR_CANNOT_READ_BLOCK;
    }
 
    // Even if those two checks shall have been done before
    TRACE("Reading DDT block header at position %" PRIu64, entry->offset);
    read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
 
    if(read_bytes != sizeof(DdtHeader2))
    {
        FATAL("Could not read block header at %" PRIu64 "", entry->offset);
 
        TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
        return AARUF_ERROR_CANNOT_READ_BLOCK;
    }
 
    ctx->image_info.ImageSize += ddt_header.cmpLength;
 
    if(entry->dataType == UserData)
    {
        // User area sectors is blocks stored in DDT minus the negative and overflow displacement blocks
        ctx->image_info.Sectors   = ddt_header.blocks - ddt_header.negative - ddt_header.overflow;
        // We need the header later for the shift calculations
        ctx->user_data_ddt_header = ddt_header;
        ctx->ddt_version          = 2;
        // Store the primary DDT table's file offset for secondary table references
        ctx->primary_ddt_offset   = entry->offset;
 
        // Check for DDT compression
        switch(ddt_header.compression)
        {
            case Lzma:
                if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
                {
                    FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
 
                cmp_data = (uint8_t *)malloc(lzma_size);
                if(cmp_data == NULL)
                {
                    TRACE("Cannot allocate memory for DDT, continuing...");
                    break;
                }
 
                buffer = malloc(ddt_header.length);
                if(buffer == NULL)
                {
                    TRACE("Cannot allocate memory for DDT, continuing...");
                    free(cmp_data);
                    break;
                }
 
                read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
                if(read_bytes != LZMA_PROPERTIES_LENGTH)
                {
                    TRACE("Could not read LZMA properties, continuing...");
                    free(cmp_data);
                    free(buffer);
                    break;
                }
 
                read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
                if(read_bytes != lzma_size)
                {
                    TRACE("Could not read compressed block, continuing...");
                    free(cmp_data);
                    free(buffer);
                    break;
                }
 
                read_bytes = ddt_header.length;
                TRACE("Decompressing block of size %zu bytes", ddt_header.length);
                error_no = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
                                                    LZMA_PROPERTIES_LENGTH);
 
                if(error_no != 0)
                {
                    FATAL("Got error %d from LZMA, stopping...", error_no);
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                if(read_bytes != ddt_header.length)
                {
                    FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                free(cmp_data);
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
 
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                ctx->user_data_ddt2 = (uint64_t *)buffer;
 
                ctx->in_memory_ddt   = true;
                *found_user_data_ddt = true;
 
                break;
            case None:
                buffer = malloc(ddt_header.length);
 
                if(buffer == NULL)
                {
                    TRACE("Cannot allocate memory for DDT, continuing...");
                    break;
                }
 
                TRACE("Reading DDT of length %zu bytes", ddt_header.length);
                read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
 
                if(read_bytes != ddt_header.length)
                {
                    free(buffer);
                    FATAL("Could not read deduplication table, continuing...");
                    break;
                }
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                ctx->user_data_ddt2 = (uint64_t *)buffer;
 
                ctx->in_memory_ddt   = true;
                *found_user_data_ddt = true;
 
                break;
            default:
                TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
                *found_user_data_ddt = false;
                break;
        }
    }
    else if(entry->dataType == CdSectorPrefix || entry->dataType == CdSectorSuffix)
        switch(ddt_header.compression)
        {
            case Lzma:
                if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
                {
                    FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
 
                cmp_data = (uint8_t *)malloc(lzma_size);
                if(cmp_data == NULL)
                {
                    TRACE("Cannot allocate memory for DDT, continuing...");
                    break;
                }
 
                buffer = malloc(ddt_header.length);
                if(buffer == NULL)
                {
                    TRACE("Cannot allocate memory for DDT, continuing...");
                    free(cmp_data);
                    break;
                }
 
                read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
                if(read_bytes != LZMA_PROPERTIES_LENGTH)
                {
                    TRACE("Could not read LZMA properties, continuing...");
                    free(cmp_data);
                    free(buffer);
                    break;
                }
 
                read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
                if(read_bytes != lzma_size)
                {
                    TRACE("Could not read compressed block, continuing...");
                    free(cmp_data);
                    free(buffer);
                    break;
                }
 
                read_bytes = ddt_header.length;
                TRACE("Decompressing block of size %zu bytes", ddt_header.length);
                error_no = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
                                                    LZMA_PROPERTIES_LENGTH);
 
                if(error_no != 0)
                {
                    FATAL("Got error %d from LZMA, stopping...", error_no);
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                if(read_bytes != ddt_header.length)
                {
                    FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                free(cmp_data);
                cmp_data = NULL;
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                if(entry->dataType == CdSectorPrefix)
                    ctx->sector_prefix_ddt2 = (uint64_t *)buffer;
                else if(entry->dataType == CdSectorSuffix)
                    ctx->sector_suffix_ddt2 = (uint64_t *)buffer;
                else
                    free(buffer);
 
                break;
 
            case None:
                buffer = malloc(ddt_header.length);
 
                if(buffer == NULL)
                {
                    TRACE("Cannot allocate memory for deduplication table.");
                    break;
                }
 
                read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
 
                if(read_bytes != ddt_header.length)
                {
                    free(buffer);
                    FATAL("Could not read deduplication table, continuing...");
                    break;
                }
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                if(entry->dataType == CdSectorPrefix)
                    ctx->sector_prefix_ddt2 = (uint64_t *)buffer;
                else if(entry->dataType == CdSectorSuffix)
                    ctx->sector_suffix_ddt2 = (uint64_t *)buffer;
                else
                    free(buffer);
 
                break;
            default:
                TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
                break;
        }
 
    TRACE("Exiting process_ddt_v2() = AARUF_STATUS_OK");
    return AARUF_STATUS_OK;
}

int32_t decode_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, bool negative, uint64_t *offset,
                            uint64_t *block_offset, uint8_t *sector_status)
{
    TRACE("Entering decode_ddt_entry_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative, *offset,
          *block_offset, *sector_status);
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
 
        TRACE("Exiting decode_ddt_entry_v2() = AARUF_ERROR_NOT_AARUFORMAT");
        return AARUF_ERROR_NOT_AARUFORMAT;
    }
 
    if(ctx->user_data_ddt_header.tableShift > 0)
        return decode_ddt_multi_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status);
 
    return decode_ddt_single_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status);
}

int32_t decode_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset,
                                   uint64_t *block_offset, uint8_t *sector_status)
{
    TRACE("Entering decode_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
          *offset, *block_offset, *sector_status);
 
    uint64_t ddt_entry = 0;
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
 
        TRACE("Exiting decode_ddt_single_level_v2() = AARUF_ERROR_NOT_AARUFORMAT");
        return AARUF_ERROR_NOT_AARUFORMAT;
    }
 
    // Should not really be here
    if(ctx->user_data_ddt_header.tableShift != 0)
    {
        FATAL("DDT table shift is not zero, but we are in single-level DDT decoding.");
        TRACE("Exiting decode_ddt_single_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
        return AARUF_ERROR_CANNOT_READ_BLOCK;
    }
 
    // Calculate positive or negative sector
    if(negative)
        sector_address = ctx->user_data_ddt_header.negative - sector_address;
    else
        sector_address += ctx->user_data_ddt_header.negative;
 
    ddt_entry = ctx->user_data_ddt2[sector_address];
 
    if(ddt_entry == 0)
    {
        *sector_status = SectorStatusNotDumped;
        *offset        = 0;
        *block_offset  = 0;
        TRACE("Exiting decode_ddt_single_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
              sector_address, *offset, *block_offset, *sector_status);
        return AARUF_STATUS_OK;
    }
 
    *sector_status = ddt_entry >> 60;
    ddt_entry &= 0xFFFFFFFFFFFFFFF;
 
    const uint64_t offset_mask = (uint64_t)((1 << ctx->user_data_ddt_header.dataShift) - 1);
    *offset                    = ddt_entry & offset_mask;
    *block_offset =
        (ddt_entry >> ctx->user_data_ddt_header.dataShift) * (1 << ctx->user_data_ddt_header.blockAlignmentShift);
 
    TRACE("Exiting decode_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
          sector_address, negative, *offset, *block_offset, *sector_status);
    return AARUF_STATUS_OK;
}

int32_t decode_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset,
                                  uint64_t *block_offset, uint8_t *sector_status)
{
    TRACE("Entering decode_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
          *offset, *block_offset, *sector_status);
 
    uint64_t   ddt_entry = 0;
    uint8_t    lzma_properties[LZMA_PROPERTIES_LENGTH];
    size_t     lzma_size            = 0;
    uint8_t   *cmp_data             = NULL;
    uint8_t   *buffer               = NULL;
    crc64_ctx *crc64_context        = NULL;
    uint64_t   crc64                = 0;
    int        items_per_ddt_entry  = 0;
    uint64_t   ddt_position         = 0;
    uint64_t   secondary_ddt_offset = 0;
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
 
        TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_NOT_AARUFORMAT");
        return AARUF_ERROR_NOT_AARUFORMAT;
    }
 
    // Should not really be here
    if(ctx->user_data_ddt_header.tableShift == 0)
    {
        FATAL("DDT table shift is zero, but we are in multi-level DDT decoding.");
        TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
        return AARUF_ERROR_CANNOT_READ_BLOCK;
    }
 
    // Calculate positive or negative sector
    if(negative)
        sector_address = ctx->user_data_ddt_header.negative - sector_address;
    else
        sector_address += ctx->user_data_ddt_header.negative;
 
    items_per_ddt_entry  = 1 << ctx->user_data_ddt_header.tableShift;
    ddt_position         = sector_address / items_per_ddt_entry;
    secondary_ddt_offset = ctx->user_data_ddt2[ddt_position];
 
    // Position in file of the child DDT table
    secondary_ddt_offset *= 1 << ctx->user_data_ddt_header.blockAlignmentShift;
 
    // Is the one we have cached the same as the one we need to read?
    if(ctx->cached_ddt_offset != secondary_ddt_offset)
    {
        int32_t error_no = 0;
        fseek(ctx->imageStream, secondary_ddt_offset, SEEK_SET);
        DdtHeader2 ddt_header;
        size_t     read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
 
        if(read_bytes != sizeof(DdtHeader2))
        {
            FATAL("Could not read block header at %" PRIu64 "", secondary_ddt_offset);
            TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
            return AARUF_ERROR_CANNOT_READ_BLOCK;
        }
 
        if((ddt_header.identifier != DeDuplicationTableSecondary &&
            ddt_header.identifier != DeDuplicationTableSAlpha) ||
           ddt_header.type != UserData)
        {
            FATAL("Invalid block header at %" PRIu64 "", secondary_ddt_offset);
            TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
            return AARUF_ERROR_CANNOT_READ_BLOCK;
        }
 
        // Check for DDT compression
        switch(ddt_header.compression)
        {
            case Lzma:
                if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
                {
                    FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
 
                cmp_data = (uint8_t *)malloc(lzma_size);
                if(cmp_data == NULL)
                {
                    FATAL("Cannot allocate memory for DDT, stopping...");
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                buffer = malloc(ddt_header.length);
                if(buffer == NULL)
                {
                    FATAL("Cannot allocate memory for DDT, stopping...");
                    free(cmp_data);
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
                if(read_bytes != LZMA_PROPERTIES_LENGTH)
                {
                    FATAL("Could not read LZMA properties, stopping...");
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
                if(read_bytes != lzma_size)
                {
                    FATAL("Could not read compressed block, stopping...");
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                TRACE("Decompressing block of size %zu bytes", ddt_header.length);
                read_bytes = ddt_header.length;
                error_no   = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
                                                      LZMA_PROPERTIES_LENGTH);
 
                if(error_no != 0)
                {
                    FATAL("Got error %d from LZMA, stopping...", error_no);
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                if(read_bytes != ddt_header.length)
                {
                    FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
                    free(cmp_data);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                }
 
                free(cmp_data);
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                // Free old cached DDT before replacing it
                free(ctx->cached_secondary_ddt2);
 
                ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
 
                ctx->cached_ddt_offset = secondary_ddt_offset;
 
                break;
            case None:
                buffer = malloc(ddt_header.length);
 
                if(buffer == NULL)
                {
                    FATAL("Cannot allocate memory for DDT, stopping...");
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
 
                if(read_bytes != ddt_header.length)
                {
                    free(buffer);
                    FATAL("Could not read deduplication table, stopping...");
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                crc64_context = aaruf_crc64_init();
 
                if(crc64_context == NULL)
                {
                    FATAL("Could not initialize CRC64.");
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                    return AARUF_ERROR_CANNOT_READ_BLOCK;
                }
 
                aaruf_crc64_update(crc64_context, buffer, read_bytes);
                aaruf_crc64_final(crc64_context, &crc64);
                aaruf_crc64_free(crc64_context);
 
                if(crc64 != ddt_header.crc64)
                {
                    FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
                    free(buffer);
                    TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                    return AARUF_ERROR_INVALID_BLOCK_CRC;
                }
 
                // Free old cached DDT before replacing it
                free(ctx->cached_secondary_ddt2);
 
                ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
 
                ctx->cached_ddt_offset = secondary_ddt_offset;
 
                break;
            default:
                FATAL("Found unknown compression type %d, stopping...", ddt_header.compression);
                TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                return AARUF_ERROR_CANNOT_READ_BLOCK;
        }
    }
 
    ddt_entry = ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry];
 
    if(ddt_entry == 0)
    {
        *sector_status = SectorStatusNotDumped;
        *offset        = 0;
        *block_offset  = 0;
 
        TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
              sector_address, *offset, *block_offset, *sector_status);
        return AARUF_STATUS_OK;
    }
 
    *sector_status = ddt_entry >> 60;
    ddt_entry &= 0x0FFFFFFFFFFFFFFF;
 
    const uint64_t offset_mask = (uint64_t)((1 << ctx->user_data_ddt_header.dataShift) - 1);
    *offset                    = ddt_entry & offset_mask;
    *block_offset =
        (ddt_entry >> ctx->user_data_ddt_header.dataShift) * (1 << ctx->user_data_ddt_header.blockAlignmentShift);
 
    TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
          sector_address, negative, *offset, *block_offset, *sector_status);
    return AARUF_STATUS_OK;
}

bool set_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, const bool negative,
                      const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status,
                      uint64_t *ddt_entry)
{
    TRACE("Entering set_ddt_entry_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative, offset,
          block_offset, sector_status);
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
        return false;
    }
 
    if(ctx->user_data_ddt_header.tableShift > 0)
        return set_ddt_multi_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status, ddt_entry);
 
    return set_ddt_single_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status, ddt_entry);
}

bool set_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, const bool negative,
                             const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status,
                             uint64_t *ddt_entry)
{
    TRACE("Entering set_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
          offset, block_offset, sector_status);
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
        TRACE("Exiting set_ddt_single_level_v2() = false");
        return false;
    }
 
    // Should not really be here
    if(ctx->user_data_ddt_header.tableShift != 0)
    {
        FATAL("DDT table shift is not zero, but we are in single-level DDT setting.");
        TRACE("Exiting set_ddt_single_level_v2() = false");
        return false;
    }
 
    // Calculate positive or negative sector
    if(negative)
        sector_address = ctx->user_data_ddt_header.negative - sector_address;
    else
        sector_address += ctx->user_data_ddt_header.negative;
 
    if(*ddt_entry == 0)
    {
        const uint64_t block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
        *ddt_entry                 = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
                     block_index << ctx->user_data_ddt_header.dataShift;
 
        // Overflow detection for DDT entry
        if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
        {
            FATAL("DDT overflow: media does not fit in big DDT");
            TRACE("Exiting set_ddt_single_level_v2() = false");
            return false;
        }
    }
 
    // Sector status can be different from previous deduplicated sector
    *ddt_entry &= 0x0FFFFFFFFFFFFFFF;
    *ddt_entry |= (uint64_t)sector_status << 60;
 
    TRACE("Setting big single-level DDT entry %d to %ull", sector_address, (uint64_t)*ddt_entry);
    ctx->user_data_ddt2[sector_address] = *ddt_entry;
    ctx->dirty_single_level_ddt         = true;  // Mark single-level DDT as dirty
 
    TRACE("Exiting set_ddt_single_level_v2() = true");
    return true;
}

bool set_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t offset,
                            uint64_t block_offset, uint8_t sector_status, uint64_t *ddt_entry)
{
    TRACE("Entering set_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %" PRIu64 ", %" PRIu64 ", %d)", ctx, sector_address,
          negative, offset, block_offset, sector_status);
 
    uint64_t   items_per_ddt_entry  = 0;
    uint64_t   ddt_position         = 0;
    uint64_t   secondary_ddt_offset = 0;
    uint64_t   block_index          = 0;
    uint8_t   *buffer               = NULL;
    crc64_ctx *crc64_context        = NULL;
    uint64_t   crc64                = 0;
    DdtHeader2 ddt_header;
    size_t     written_bytes    = 0;
    long       end_of_file      = 0;
    bool       create_new_table = false;
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
        TRACE("Exiting set_ddt_multi_level_v2() = false");
        return false;
    }
 
    // Should not really be here
    if(ctx->user_data_ddt_header.tableShift == 0)
    {
        FATAL("DDT table shift is zero, but we are in multi-level DDT setting.");
        TRACE("Exiting set_ddt_multi_level_v2() = false");
        return false;
    }
 
    // Calculate positive or negative sector
    if(negative)
        sector_address = ctx->user_data_ddt_header.negative - sector_address;
    else
        sector_address += ctx->user_data_ddt_header.negative;
 
    // Step 1: Calculate the corresponding secondary level table
    items_per_ddt_entry  = 1 << ctx->user_data_ddt_header.tableShift;
    ddt_position         = sector_address / items_per_ddt_entry;
    secondary_ddt_offset = ctx->user_data_ddt2[ddt_position];
 
    // Position in file of the child DDT table
    secondary_ddt_offset *= 1 << ctx->user_data_ddt_header.blockAlignmentShift;
 
    // Step 2: Check if it corresponds to the currently in-memory cached secondary level table
    if(ctx->cached_ddt_offset == secondary_ddt_offset && secondary_ddt_offset != 0)
    {
        // Update the corresponding DDT entry directly in the cached table
        if(*ddt_entry == 0)
        {
            block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
            *ddt_entry  = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
                         block_index << ctx->user_data_ddt_header.dataShift;
 
            // Overflow detection for DDT entry
            if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
            {
                FATAL("DDT overflow: media does not fit in big DDT");
                TRACE("Exiting set_ddt_multi_level_v2() = false");
                return false;
            }
        }
 
        // Sector status can be different from previous deduplicated sector
        *ddt_entry &= 0x0FFFFFFFFFFFFFFF;
        *ddt_entry |= (uint64_t)sector_status << 60;
 
        TRACE("Setting small secondary DDT entry %d to %ull", sector_address % items_per_ddt_entry,
              (uint64_t)*ddt_entry);
        ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry] = *ddt_entry;
        ctx->dirty_secondary_ddt                                         = true;  // Mark secondary DDT as dirty
 
        TRACE("Updated cached secondary DDT entry at position %" PRIu64, sector_address % items_per_ddt_entry);
        TRACE("Exiting set_ddt_multi_level_v2() = true");
        return true;
    }
 
    // Step 2.5: Handle case where we have a cached secondary DDT that has never been written to disk
    // but does not contain the requested block
    if(ctx->cached_ddt_offset == 0 && (ctx->cached_secondary_ddt2 != NULL))
    {
        // Only write the cached table to disk if the requested block belongs to a different DDT position
        if(ddt_position != ctx->cached_ddt_position)
        {
            TRACE("Current secondary DDT in memory belongs to position %" PRIu64
                  " but requested block needs position %" PRIu64,
                  ctx->cached_ddt_position, ddt_position);
 
            // Write the cached DDT to disk before proceeding with the new one
 
            // Close the current data block first
            if(ctx->writing_buffer != NULL) aaruf_close_current_block(ctx);
 
            // Get current position and seek to end of file
            fseek(ctx->imageStream, 0, SEEK_END);
            end_of_file = ftell(ctx->imageStream);
 
            // Align to block boundary
            uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
            end_of_file             = end_of_file + alignment_mask & ~alignment_mask;
            fseek(ctx->imageStream, end_of_file, SEEK_SET);
 
            // Prepare DDT header for the never-written cached table
            memset(&ddt_header, 0, sizeof(DdtHeader2));
            ddt_header.identifier  = DeDuplicationTableSecondary;
            ddt_header.type        = UserData;
            ddt_header.compression = ctx->compression_enabled ? Lzma : None;  // Use no compression for simplicity
            ddt_header.levels      = ctx->user_data_ddt_header.levels;
            ddt_header.tableLevel  = ctx->user_data_ddt_header.tableLevel + 1;
            ddt_header.previousLevelOffset = ctx->primary_ddt_offset;
            ddt_header.negative            = ctx->user_data_ddt_header.negative;
            ddt_header.blocks              = items_per_ddt_entry;
            ddt_header.overflow            = ctx->user_data_ddt_header.overflow;
            ddt_header.start = ctx->cached_ddt_position * items_per_ddt_entry;  // Use cached position with table shift
            ddt_header.blockAlignmentShift = ctx->user_data_ddt_header.blockAlignmentShift;
            ddt_header.dataShift           = ctx->user_data_ddt_header.dataShift;
            ddt_header.tableShift          = 0;  // Secondary tables are single level
            ddt_header.entries             = items_per_ddt_entry;
 
            // Calculate data size
 
            ddt_header.length = items_per_ddt_entry * sizeof(uint64_t);
 
            // Calculate CRC64 of the data
            crc64_context = aaruf_crc64_init();
            if(crc64_context == NULL)
            {
                FATAL("Could not initialize CRC64.");
                TRACE("Exiting set_ddt_multi_level_v2() = false");
                return false;
            }
 
            aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cached_secondary_ddt2, (uint32_t)ddt_header.length);
 
            aaruf_crc64_final(crc64_context, &crc64);
            aaruf_crc64_free(crc64_context);
            ddt_header.crc64 = crc64;
 
            uint8_t *cmp_buffer                              = NULL;
            uint8_t  lzma_properties[LZMA_PROPERTIES_LENGTH] = {0};
 
            if(ddt_header.compression == None)
            {
 
                cmp_buffer          = (uint8_t *)ctx->cached_secondary_ddt2;
                ddt_header.cmpCrc64 = ddt_header.crc64;
            }
            else
            {
                cmp_buffer = malloc((size_t)ddt_header.length * 2);  // Allocate double size for compression
                if(cmp_buffer == NULL)
                {
                    TRACE("Failed to allocate memory for secondary DDT v2 compression");
                    return AARUF_ERROR_NOT_ENOUGH_MEMORY;
                }
 
                size_t dst_size   = (size_t)ddt_header.length * 2 * 2;
                size_t props_size = LZMA_PROPERTIES_LENGTH;
                aaruf_lzma_encode_buffer(cmp_buffer, &dst_size, (uint8_t *)ctx->cached_secondary_ddt2,
                                         ddt_header.length, lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0,
                                         2, 273, 8);
 
                ddt_header.cmpLength = (uint32_t)dst_size;
 
                if(ddt_header.cmpLength >= ddt_header.length)
                {
                    ddt_header.compression = None;
                    free(cmp_buffer);
 
                    cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
                }
            }
 
            if(ddt_header.compression == None)
            {
                ddt_header.cmpLength = ddt_header.length;
                ddt_header.cmpCrc64  = ddt_header.crc64;
            }
            else
                ddt_header.cmpCrc64 = aaruf_crc64_data(cmp_buffer, (uint32_t)ddt_header.cmpLength);
 
            if(ddt_header.compression == Lzma) ddt_header.cmpLength += LZMA_PROPERTIES_LENGTH;
 
            // Write header
            written_bytes = fwrite(&ddt_header, sizeof(DdtHeader2), 1, ctx->imageStream);
            if(written_bytes != 1)
            {
                FATAL("Could not write never-written DDT header to file.");
                TRACE("Exiting set_ddt_multi_level_v2() = false");
                return false;
            }
 
            // Write data
            if(ddt_header.compression == Lzma) fwrite(lzma_properties, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream);
 
            if(fwrite(cmp_buffer, ddt_header.cmpLength, 1, ctx->imageStream) != 1)
            {
                FATAL("Could not write never-written DDT data to file.");
                TRACE("Exiting set_ddt_multi_level_v2() = false");
                return false;
            }
 
            if(ddt_header.compression == Lzma) free(cmp_buffer);
 
            // Add index entry for the newly written secondary DDT
            IndexEntry new_ddt_entry;
            new_ddt_entry.blockType = DeDuplicationTableSecondary;
            new_ddt_entry.dataType  = UserData;
            new_ddt_entry.offset    = end_of_file;
 
            utarray_push_back(ctx->index_entries, &new_ddt_entry);
            ctx->dirty_index_block = true;
            TRACE("Added new DDT index entry for never-written table at offset %" PRIu64, end_of_file);
 
            // Update the primary level table entry to point to the new location of the secondary table
            uint64_t new_secondary_table_block_offset = end_of_file >> ctx->user_data_ddt_header.blockAlignmentShift;
 
            ctx->user_data_ddt2[ctx->cached_ddt_position] = new_secondary_table_block_offset;
            ctx->dirty_primary_ddt                        = true;  // Mark primary DDT as dirty
 
            // Write the updated primary table back to its original position in the file
            long saved_pos = ftell(ctx->imageStream);
            fseek(ctx->imageStream, ctx->primary_ddt_offset + sizeof(DdtHeader2), SEEK_SET);
 
            size_t primary_table_size = ctx->user_data_ddt_header.entries * sizeof(uint64_t);
 
            written_bytes = fwrite(ctx->user_data_ddt2, primary_table_size, 1, ctx->imageStream);
 
            if(written_bytes != 1)
            {
                FATAL("Could not flush primary DDT table to file after writing never-written secondary table.");
                TRACE("Exiting set_ddt_multi_level_v2() = false");
                return false;
            }
 
            // Update nextBlockPosition to ensure future blocks don't overwrite the DDT
            uint64_t ddt_total_size  = sizeof(DdtHeader2) + ddt_header.length;
            ctx->next_block_position = end_of_file + ddt_total_size + alignment_mask & ~alignment_mask;
            block_offset             = ctx->next_block_position;
            offset                   = 0;
            TRACE("Updated nextBlockPosition after never-written DDT write to %" PRIu64, ctx->next_block_position);
 
            // Free the cached table
 
            free(ctx->cached_secondary_ddt2);
            ctx->cached_secondary_ddt2 = NULL;
 
            // Reset cached values since we've written and freed the table
            ctx->cached_ddt_offset   = 0;
            ctx->cached_ddt_position = 0;
 
            // Restore file position
            fseek(ctx->imageStream, saved_pos, SEEK_SET);
 
            TRACE("Successfully wrote never-written cached secondary DDT to disk");
        }
        else
            // The cached DDT is actually for the requested block range, so we can use it directly
            TRACE("Cached DDT is for the correct block range, using it directly");
        // No need to write to disk, just continue with the cached table
    }
 
    // Step 3: Write the currently in-memory cached secondary level table to the end of the file
    if(ctx->cached_ddt_offset != 0)
    {
        long current_pos = 0;
        // Close the current data block first
        if(ctx->writing_buffer != NULL) aaruf_close_current_block(ctx);
 
        // Get current position and seek to end of file
        current_pos = ftell(ctx->imageStream);
        fseek(ctx->imageStream, 0, SEEK_END);
        end_of_file = ftell(ctx->imageStream);
 
        // Align to block boundary
        uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
        end_of_file             = end_of_file + alignment_mask & ~alignment_mask;
        fseek(ctx->imageStream, end_of_file, SEEK_SET);
 
        // Prepare DDT header for the cached table
        memset(&ddt_header, 0, sizeof(DdtHeader2));
        ddt_header.identifier          = DeDuplicationTableSecondary;
        ddt_header.type                = UserData;
        ddt_header.compression         = ctx->compression_enabled ? Lzma : None;
        ddt_header.levels              = ctx->user_data_ddt_header.levels;
        ddt_header.tableLevel          = ctx->user_data_ddt_header.tableLevel + 1;
        ddt_header.previousLevelOffset = ctx->primary_ddt_offset;  // Set to primary DDT table location
        ddt_header.negative            = ctx->user_data_ddt_header.negative;
        ddt_header.blocks              = items_per_ddt_entry;
        ddt_header.overflow            = ctx->user_data_ddt_header.overflow;
        ddt_header.start               = ddt_position * items_per_ddt_entry;  // First block this DDT table references
        ddt_header.blockAlignmentShift = ctx->user_data_ddt_header.blockAlignmentShift;
        ddt_header.dataShift           = ctx->user_data_ddt_header.dataShift;
        ddt_header.tableShift          = 0;  // Secondary tables are single level
        ddt_header.entries             = items_per_ddt_entry;
 
        // Calculate data size
 
        ddt_header.length = items_per_ddt_entry * sizeof(uint64_t);
 
        // Calculate CRC64 of the data
        crc64_context = aaruf_crc64_init();
        if(crc64_context == NULL)
        {
            FATAL("Could not initialize CRC64.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cached_secondary_ddt2, ddt_header.length);
 
        aaruf_crc64_final(crc64_context, &crc64);
        aaruf_crc64_free(crc64_context);
        ddt_header.crc64 = crc64;
 
        uint8_t *cmp_buffer                              = NULL;
        uint8_t  lzma_properties[LZMA_PROPERTIES_LENGTH] = {0};
 
        if(ddt_header.compression == None)
        {
 
            cmp_buffer          = (uint8_t *)ctx->cached_secondary_ddt2;
            ddt_header.cmpCrc64 = ddt_header.crc64;
        }
        else
        {
            cmp_buffer = malloc((size_t)ddt_header.length * 2);  // Allocate double size for compression
            if(cmp_buffer == NULL)
            {
                TRACE("Failed to allocate memory for secondary DDT v2 compression");
                return AARUF_ERROR_NOT_ENOUGH_MEMORY;
            }
 
            size_t dst_size   = (size_t)ddt_header.length * 2 * 2;
            size_t props_size = LZMA_PROPERTIES_LENGTH;
            aaruf_lzma_encode_buffer(cmp_buffer, &dst_size, (uint8_t *)ctx->cached_secondary_ddt2, ddt_header.length,
                                     lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
 
            ddt_header.cmpLength = (uint32_t)dst_size;
 
            if(ddt_header.cmpLength >= ddt_header.length)
            {
                ddt_header.compression = None;
                free(cmp_buffer);
 
                cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
            }
        }
 
        if(ddt_header.compression == None)
        {
            ddt_header.cmpLength = ddt_header.length;
            ddt_header.cmpCrc64  = ddt_header.crc64;
        }
        else
            ddt_header.cmpCrc64 = aaruf_crc64_data(cmp_buffer, (uint32_t)ddt_header.cmpLength);
 
        if(ddt_header.compression == Lzma) ddt_header.cmpLength += LZMA_PROPERTIES_LENGTH;
 
        // Write header
        if(ddt_header.compression == Lzma) fwrite(lzma_properties, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream);
 
        written_bytes = fwrite(&ddt_header, sizeof(DdtHeader2), 1, ctx->imageStream);
        if(written_bytes != 1)
        {
            FATAL("Could not write DDT header to file.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        // Write data
        written_bytes = fwrite(cmp_buffer, ddt_header.cmpLength, 1, ctx->imageStream);
 
        if(written_bytes != 1)
        {
            FATAL("Could not write DDT data to file.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        if(ddt_header.compression == Lzma) free(cmp_buffer);
 
        // Update index: remove old entry and add new one for the evicted secondary DDT
        TRACE("Updating index for evicted secondary DDT");
 
        // Remove old index entry for the cached DDT
        if(ctx->cached_ddt_offset != 0)
        {
            TRACE("Removing old index entry for DDT at offset %" PRIu64, ctx->cached_ddt_offset);
            IndexEntry *entry = NULL;
 
            // Find and remove the old index entry
            for(unsigned int i = 0; i < utarray_len(ctx->index_entries); i++)
            {
                entry = (IndexEntry *)utarray_eltptr(ctx->index_entries, i);
                if(entry && entry->offset == ctx->cached_ddt_offset &&
                   (entry->blockType == DeDuplicationTableSecondary || entry->blockType == DeDuplicationTableSAlpha))
                {
                    TRACE("Found old DDT index entry at position %u, removing", i);
                    utarray_erase(ctx->index_entries, i, 1);
                    break;
                }
            }
        }
 
        // Add new index entry for the newly written secondary DDT
        IndexEntry new_ddt_entry;
        new_ddt_entry.blockType = DeDuplicationTableSecondary;
        new_ddt_entry.dataType  = UserData;
        new_ddt_entry.offset    = end_of_file;
 
        utarray_push_back(ctx->index_entries, &new_ddt_entry);
        ctx->dirty_index_block = true;
        TRACE("Added new DDT index entry at offset %" PRIu64, end_of_file);
 
        // Step 4: Update the primary level table entry and flush it back to file
        uint64_t new_secondary_table_block_offset = end_of_file >> ctx->user_data_ddt_header.blockAlignmentShift;
 
        // Update the primary table entry to point to the new location of the secondary table
        // Use ddtPosition which was calculated from sectorAddress, not cachedDdtOffset
 
        ctx->user_data_ddt2[ddt_position] = new_secondary_table_block_offset;
        ctx->dirty_primary_ddt            = true;  // Mark primary DDT as dirty
 
        // Write the updated primary table back to its original position in the file
        long saved_pos = ftell(ctx->imageStream);
        fseek(ctx->imageStream, ctx->primary_ddt_offset + sizeof(DdtHeader2), SEEK_SET);
 
        size_t primary_table_size = ctx->user_data_ddt_header.entries * sizeof(uint64_t);
 
        written_bytes = fwrite(ctx->user_data_ddt2, primary_table_size, 1, ctx->imageStream);
 
        if(written_bytes != 1)
        {
            FATAL("Could not flush primary DDT table to file.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        // Update nextBlockPosition to ensure future blocks don't overwrite the DDT
        uint64_t ddt_total_size  = sizeof(DdtHeader2) + ddt_header.length;
        ctx->next_block_position = end_of_file + ddt_total_size + alignment_mask & ~alignment_mask;
        block_offset             = ctx->next_block_position;
        offset                   = 0;
        TRACE("Updated nextBlockPosition after DDT write to %" PRIu64, ctx->next_block_position);
 
        fseek(ctx->imageStream, saved_pos, SEEK_SET);
 
        // Free the cached table
 
        free(ctx->cached_secondary_ddt2);
        ctx->cached_secondary_ddt2 = NULL;
 
        // Restore file position
        fseek(ctx->imageStream, current_pos, SEEK_SET);
    }
 
    // Step 5: Check if the specified block already has an existing secondary level table
    create_new_table = ctx->cached_secondary_ddt2 == NULL;
 
    if(!create_new_table && secondary_ddt_offset != 0)
    {
        // Load existing table
        fseek(ctx->imageStream, secondary_ddt_offset, SEEK_SET);
        size_t read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
 
        if(read_bytes != sizeof(DdtHeader2) || ddt_header.identifier != DeDuplicationTable2 ||
           ddt_header.type != UserData)
        {
            FATAL("Invalid secondary DDT header at %" PRIu64, secondary_ddt_offset);
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        // Read the table data (assuming no compression for now)
        buffer = malloc(ddt_header.length);
        if(buffer == NULL)
        {
            FATAL("Cannot allocate memory for secondary DDT.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
        if(read_bytes != ddt_header.length)
        {
            FATAL("Could not read secondary DDT data.");
            free(buffer);
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        // Verify CRC
        crc64_context = aaruf_crc64_init();
        if(crc64_context == NULL)
        {
            FATAL("Could not initialize CRC64.");
            free(buffer);
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        aaruf_crc64_update(crc64_context, buffer, read_bytes);
        aaruf_crc64_final(crc64_context, &crc64);
        aaruf_crc64_free(crc64_context);
 
        if(crc64 != ddt_header.crc64)
        {
            FATAL("Secondary DDT CRC mismatch. Expected 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
            free(buffer);
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        // Cache the loaded table
 
        ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
 
        ctx->cached_ddt_offset = secondary_ddt_offset;
    }
 
    if(create_new_table)
    {
        // Create a new empty table
        size_t table_size = items_per_ddt_entry * sizeof(uint64_t);
 
        buffer = calloc(1, table_size);
        if(buffer == NULL)
        {
            FATAL("Cannot allocate memory for new secondary DDT.");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
 
        ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
 
        ctx->cached_ddt_offset   = 0;             // Will be set when written to file
        ctx->cached_ddt_position = ddt_position;  // Track which primary DDT position this new table belongs to
        TRACE("Created new secondary DDT for position %" PRIu64, ddt_position);
    }
 
    // Step 6: Update the corresponding DDT entry
    if(*ddt_entry == 0)
    {
        block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
        *ddt_entry  = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
                     block_index << ctx->user_data_ddt_header.dataShift;
 
        // Overflow detection for DDT entry
        if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
        {
            FATAL("DDT overflow: media does not fit in big DDT");
            TRACE("Exiting set_ddt_multi_level_v2() = false");
            return false;
        }
    }
 
    // Sector status can be different from previous deduplicated sector
    *ddt_entry &= 0x0FFFFFFFFFFFFFFF;
    *ddt_entry |= (uint64_t)sector_status << 60;
 
    TRACE("Setting big secondary DDT entry %d to %ull", sector_address % items_per_ddt_entry, (uint64_t)*ddt_entry);
    ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry] = *ddt_entry;
    ctx->dirty_secondary_ddt                                         = true;
 
    TRACE("Updated secondary DDT entry at position %" PRIu64, sector_address % items_per_ddt_entry);
    TRACE("Exiting set_ddt_multi_level_v2() = true");
    return true;
}

bool set_ddt_tape(aaruformat_context *ctx, uint64_t sector_address, const uint64_t offset, const uint64_t block_offset,
                  const uint8_t sector_status, uint64_t *ddt_entry)
{
    TRACE("Entering set_ddt_tape(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sector_address, offset, block_offset,
          sector_status);
 
    // Check if the context and image stream are valid
    if(ctx == NULL || ctx->imageStream == NULL)
    {
        FATAL("Invalid context or image stream.");
        TRACE("Exiting set_ddt_tape() = false");
        return false;
    }
 
    // Should not really be here
    if(!ctx->is_tape)
    {
        FATAL("Image is not tape, wrong function called.");
        TRACE("Exiting set_ddt_tape() = false");
        return false;
    }
 
    if(*ddt_entry == 0)
    {
        const uint64_t block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
        *ddt_entry                 = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
                     block_index << ctx->user_data_ddt_header.dataShift;
        // Overflow detection for DDT entry
        if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
        {
            FATAL("DDT overflow: media does not fit in big DDT");
            TRACE("Exiting set_ddt_tape() = false");
            return false;
        }
 
        *ddt_entry |= (uint64_t)sector_status << 60;
    }
 
    // Create DDT hash entry
    TapeDdtHashEntry *new_entry = calloc(1, sizeof(TapeDdtHashEntry));
    TapeDdtHashEntry *old_entry = NULL;
    if(new_entry == NULL)
    {
        FATAL("Cannot allocate memory for new tape DDT hash entry.");
        TRACE("Exiting set_ddt_tape() = false");
        return false;
    }
 
    TRACE("Setting tape DDT entry %d to %u", sector_address, (uint32_t)*ddt_entry);
 
    new_entry->key   = sector_address;
    new_entry->value = *ddt_entry;
 
    // Insert entry into tape DDT
    HASH_REPLACE(hh, ctx->tape_ddt, key, sizeof(uint64_t), new_entry, old_entry);
    ctx->dirty_tape_ddt = true;  // Mark tape DDT as dirty
    if(old_entry) free(old_entry);
 
    TRACE("Exiting set_ddt_tape() = true");
    return true;
}