Loading...
Searching...
No Matches
ddt_v2.c
Go to the documentation of this file.
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2026 Natalia Portillo.
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include <inttypes.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "aaruformat.h"
#include "internal.h"
#include "log.h"
int32_t process_ddt_v2(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
{
TRACE("Entering process_ddt_v2(%p, %p, %d)", ctx, entry, *found_user_data_ddt);
int pos = 0;
size_t read_bytes = 0;
DdtHeader2 ddt_header;
uint8_t *cmp_data = NULL;
uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH];
size_t lzma_size = 0;
int error_no = 0;
crc64_ctx *crc64_context = NULL;
uint64_t crc64 = 0;
uint8_t *buffer = NULL;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_NOT_AARUFORMAT");
return AARUF_ERROR_NOT_AARUFORMAT;
}
// Seek to block
pos = fseek(ctx->imageStream, entry->offset, SEEK_SET);
if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
{
FATAL("Could not seek to %" PRIu64 " as indicated by index entry...", entry->offset);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
// Even if those two checks shall have been done before
TRACE("Reading DDT block header at position %" PRIu64, entry->offset);
read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
if(read_bytes != sizeof(DdtHeader2))
{
FATAL("Could not read block header at %" PRIu64 "", entry->offset);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
ctx->image_info.ImageSize += ddt_header.cmpLength;
if(entry->dataType == UserData)
{
// User area sectors is blocks stored in DDT minus the negative and overflow displacement blocks
ctx->image_info.Sectors = ddt_header.blocks - ddt_header.negative - ddt_header.overflow;
// We need the header later for the shift calculations
ctx->user_data_ddt_header = ddt_header;
ctx->ddt_version = 2;
// Store the primary DDT table's file offset for secondary table references
ctx->primary_ddt_offset = entry->offset;
// Check for DDT compression
switch(ddt_header.compression)
{
case Lzma:
if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
{
FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
cmp_data = (uint8_t *)malloc(lzma_size);
if(cmp_data == NULL)
{
TRACE("Cannot allocate memory for DDT, continuing...");
break;
}
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
TRACE("Cannot allocate memory for DDT, continuing...");
free(cmp_data);
break;
}
read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
if(read_bytes != LZMA_PROPERTIES_LENGTH)
{
TRACE("Could not read LZMA properties, continuing...");
free(cmp_data);
free(buffer);
break;
}
read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
if(read_bytes != lzma_size)
{
TRACE("Could not read compressed block, continuing...");
free(cmp_data);
free(buffer);
break;
}
read_bytes = ddt_header.length;
TRACE("Decompressing block of size %zu bytes", ddt_header.length);
error_no = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
LZMA_PROPERTIES_LENGTH);
if(error_no != 0)
{
FATAL("Got error %d from LZMA, stopping...", error_no);
free(cmp_data);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(read_bytes != ddt_header.length)
{
FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
free(cmp_data);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
free(cmp_data);
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
ctx->user_data_ddt2 = (uint64_t *)buffer;
ctx->in_memory_ddt = true;
*found_user_data_ddt = true;
break;
case None:
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
TRACE("Cannot allocate memory for DDT, continuing...");
break;
}
TRACE("Reading DDT of length %zu bytes", ddt_header.length);
read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
if(read_bytes != ddt_header.length)
{
free(buffer);
FATAL("Could not read deduplication table, continuing...");
break;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
ctx->user_data_ddt2 = (uint64_t *)buffer;
ctx->in_memory_ddt = true;
*found_user_data_ddt = true;
break;
default:
TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
*found_user_data_ddt = false;
break;
}
}
else if(entry->dataType == CdSectorPrefix || entry->dataType == CdSectorSuffix)
switch(ddt_header.compression)
{
case Lzma:
if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
{
FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
cmp_data = (uint8_t *)malloc(lzma_size);
if(cmp_data == NULL)
{
TRACE("Cannot allocate memory for DDT, continuing...");
break;
}
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
TRACE("Cannot allocate memory for DDT, continuing...");
free(cmp_data);
break;
}
read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
if(read_bytes != LZMA_PROPERTIES_LENGTH)
{
TRACE("Could not read LZMA properties, continuing...");
free(cmp_data);
free(buffer);
break;
}
read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
if(read_bytes != lzma_size)
{
TRACE("Could not read compressed block, continuing...");
free(cmp_data);
free(buffer);
break;
}
read_bytes = ddt_header.length;
TRACE("Decompressing block of size %zu bytes", ddt_header.length);
error_no = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
LZMA_PROPERTIES_LENGTH);
if(error_no != 0)
{
FATAL("Got error %d from LZMA, stopping...", error_no);
free(cmp_data);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(read_bytes != ddt_header.length)
{
FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
free(cmp_data);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
free(cmp_data);
cmp_data = NULL;
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
if(entry->dataType == CdSectorPrefix)
ctx->sector_prefix_ddt2 = (uint64_t *)buffer;
else if(entry->dataType == CdSectorSuffix)
ctx->sector_suffix_ddt2 = (uint64_t *)buffer;
else
free(buffer);
break;
case None:
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
TRACE("Cannot allocate memory for deduplication table.");
break;
}
read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
if(read_bytes != ddt_header.length)
{
free(buffer);
FATAL("Could not read deduplication table, continuing...");
break;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
if(entry->dataType == CdSectorPrefix)
ctx->sector_prefix_ddt2 = (uint64_t *)buffer;
else if(entry->dataType == CdSectorSuffix)
ctx->sector_suffix_ddt2 = (uint64_t *)buffer;
else
free(buffer);
break;
default:
TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
break;
}
TRACE("Exiting process_ddt_v2() = AARUF_STATUS_OK");
return AARUF_STATUS_OK;
}
int32_t decode_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, bool negative, uint64_t *offset,
uint64_t *block_offset, uint8_t *sector_status)
{
TRACE("Entering decode_ddt_entry_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative, *offset,
*block_offset, *sector_status);
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting decode_ddt_entry_v2() = AARUF_ERROR_NOT_AARUFORMAT");
return AARUF_ERROR_NOT_AARUFORMAT;
}
if(ctx->user_data_ddt_header.tableShift > 0)
return decode_ddt_multi_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status);
return decode_ddt_single_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status);
}
int32_t decode_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset,
uint64_t *block_offset, uint8_t *sector_status)
{
TRACE("Entering decode_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
*offset, *block_offset, *sector_status);
uint64_t ddt_entry = 0;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting decode_ddt_single_level_v2() = AARUF_ERROR_NOT_AARUFORMAT");
return AARUF_ERROR_NOT_AARUFORMAT;
}
// Should not really be here
if(ctx->user_data_ddt_header.tableShift != 0)
{
FATAL("DDT table shift is not zero, but we are in single-level DDT decoding.");
TRACE("Exiting decode_ddt_single_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
// Calculate positive or negative sector
if(negative)
sector_address = ctx->user_data_ddt_header.negative - sector_address;
else
sector_address += ctx->user_data_ddt_header.negative;
ddt_entry = ctx->user_data_ddt2[sector_address];
if(ddt_entry == 0)
{
*sector_status = SectorStatusNotDumped;
*offset = 0;
*block_offset = 0;
TRACE("Exiting decode_ddt_single_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
sector_address, *offset, *block_offset, *sector_status);
return AARUF_STATUS_OK;
}
*sector_status = ddt_entry >> 60;
ddt_entry &= 0xFFFFFFFFFFFFFFF;
const uint64_t offset_mask = (uint64_t)((1 << ctx->user_data_ddt_header.dataShift) - 1);
*offset = ddt_entry & offset_mask;
*block_offset =
(ddt_entry >> ctx->user_data_ddt_header.dataShift) * (1 << ctx->user_data_ddt_header.blockAlignmentShift);
TRACE("Exiting decode_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
sector_address, negative, *offset, *block_offset, *sector_status);
return AARUF_STATUS_OK;
}
int32_t decode_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset,
uint64_t *block_offset, uint8_t *sector_status)
{
TRACE("Entering decode_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
*offset, *block_offset, *sector_status);
uint64_t ddt_entry = 0;
uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH];
size_t lzma_size = 0;
uint8_t *cmp_data = NULL;
uint8_t *buffer = NULL;
crc64_ctx *crc64_context = NULL;
uint64_t crc64 = 0;
int items_per_ddt_entry = 0;
uint64_t ddt_position = 0;
uint64_t secondary_ddt_offset = 0;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_NOT_AARUFORMAT");
return AARUF_ERROR_NOT_AARUFORMAT;
}
// Should not really be here
if(ctx->user_data_ddt_header.tableShift == 0)
{
FATAL("DDT table shift is zero, but we are in multi-level DDT decoding.");
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
// Calculate positive or negative sector
if(negative)
sector_address = ctx->user_data_ddt_header.negative - sector_address;
else
sector_address += ctx->user_data_ddt_header.negative;
items_per_ddt_entry = 1 << ctx->user_data_ddt_header.tableShift;
ddt_position = sector_address / items_per_ddt_entry;
secondary_ddt_offset = ctx->user_data_ddt2[ddt_position];
// Position in file of the child DDT table
secondary_ddt_offset *= 1 << ctx->user_data_ddt_header.blockAlignmentShift;
// Is the one we have cached the same as the one we need to read?
if(ctx->cached_ddt_offset != secondary_ddt_offset)
{
int32_t error_no = 0;
fseek(ctx->imageStream, secondary_ddt_offset, SEEK_SET);
DdtHeader2 ddt_header;
size_t read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
if(read_bytes != sizeof(DdtHeader2))
{
FATAL("Could not read block header at %" PRIu64 "", secondary_ddt_offset);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
if((ddt_header.identifier != DeDuplicationTableSecondary &&
ddt_header.identifier != DeDuplicationTableSAlpha) ||
ddt_header.type != UserData)
{
FATAL("Invalid block header at %" PRIu64 "", secondary_ddt_offset);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
// Check for DDT compression
switch(ddt_header.compression)
{
case Lzma:
if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
{
FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
lzma_size = (size_t)(ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH);
cmp_data = (uint8_t *)malloc(lzma_size);
if(cmp_data == NULL)
{
FATAL("Cannot allocate memory for DDT, stopping...");
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
FATAL("Cannot allocate memory for DDT, stopping...");
free(cmp_data);
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
if(read_bytes != LZMA_PROPERTIES_LENGTH)
{
FATAL("Could not read LZMA properties, stopping...");
free(cmp_data);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
if(read_bytes != lzma_size)
{
FATAL("Could not read compressed block, stopping...");
free(cmp_data);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
TRACE("Decompressing block of size %zu bytes", ddt_header.length);
read_bytes = ddt_header.length;
error_no = aaruf_lzma_decode_buffer(buffer, &read_bytes, cmp_data, &lzma_size, lzma_properties,
LZMA_PROPERTIES_LENGTH);
if(error_no != 0)
{
FATAL("Got error %d from LZMA, stopping...", error_no);
free(cmp_data);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(read_bytes != ddt_header.length)
{
FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
free(cmp_data);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
free(cmp_data);
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
// Free old cached DDT before replacing it
free(ctx->cached_secondary_ddt2);
ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
ctx->cached_ddt_offset = secondary_ddt_offset;
break;
case None:
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
FATAL("Cannot allocate memory for DDT, stopping...");
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
if(read_bytes != ddt_header.length)
{
free(buffer);
FATAL("Could not read deduplication table, stopping...");
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
// Free old cached DDT before replacing it
free(ctx->cached_secondary_ddt2);
ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
ctx->cached_ddt_offset = secondary_ddt_offset;
break;
default:
FATAL("Found unknown compression type %d, stopping...", ddt_header.compression);
TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
}
ddt_entry = ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry];
if(ddt_entry == 0)
{
*sector_status = SectorStatusNotDumped;
*offset = 0;
*block_offset = 0;
TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
sector_address, *offset, *block_offset, *sector_status);
return AARUF_STATUS_OK;
}
*sector_status = ddt_entry >> 60;
ddt_entry &= 0x0FFFFFFFFFFFFFFF;
const uint64_t offset_mask = (uint64_t)((1 << ctx->user_data_ddt_header.dataShift) - 1);
*offset = ddt_entry & offset_mask;
*block_offset =
(ddt_entry >> ctx->user_data_ddt_header.dataShift) * (1 << ctx->user_data_ddt_header.blockAlignmentShift);
TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d) = AARUF_STATUS_OK", ctx,
sector_address, negative, *offset, *block_offset, *sector_status);
return AARUF_STATUS_OK;
}
bool set_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, const bool negative,
const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status,
uint64_t *ddt_entry)
{
TRACE("Entering set_ddt_entry_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative, offset,
block_offset, sector_status);
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
return false;
}
if(ctx->user_data_ddt_header.tableShift > 0)
return set_ddt_multi_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status, ddt_entry);
return set_ddt_single_level_v2(ctx, sector_address, negative, offset, block_offset, sector_status, ddt_entry);
}
bool set_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, const bool negative,
const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status,
uint64_t *ddt_entry)
{
TRACE("Entering set_ddt_single_level_v2(%p, %" PRIu64 ", %d, %llu, %llu, %d)", ctx, sector_address, negative,
offset, block_offset, sector_status);
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting set_ddt_single_level_v2() = false");
return false;
}
// Should not really be here
if(ctx->user_data_ddt_header.tableShift != 0)
{
FATAL("DDT table shift is not zero, but we are in single-level DDT setting.");
TRACE("Exiting set_ddt_single_level_v2() = false");
return false;
}
// Calculate positive or negative sector
if(negative)
sector_address = ctx->user_data_ddt_header.negative - sector_address;
else
sector_address += ctx->user_data_ddt_header.negative;
if(*ddt_entry == 0)
{
const uint64_t block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
*ddt_entry = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
block_index << ctx->user_data_ddt_header.dataShift;
// Overflow detection for DDT entry
if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
{
FATAL("DDT overflow: media does not fit in big DDT");
TRACE("Exiting set_ddt_single_level_v2() = false");
return false;
}
}
// Sector status can be different from previous deduplicated sector
*ddt_entry &= 0x0FFFFFFFFFFFFFFF;
*ddt_entry |= (uint64_t)sector_status << 60;
TRACE("Setting big single-level DDT entry %d to %ull", sector_address, (uint64_t)*ddt_entry);
ctx->user_data_ddt2[sector_address] = *ddt_entry;
ctx->dirty_single_level_ddt = true; // Mark single-level DDT as dirty
TRACE("Exiting set_ddt_single_level_v2() = true");
return true;
}
bool set_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t offset,
uint64_t block_offset, uint8_t sector_status, uint64_t *ddt_entry)
{
TRACE("Entering set_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %" PRIu64 ", %" PRIu64 ", %d)", ctx, sector_address,
negative, offset, block_offset, sector_status);
uint64_t items_per_ddt_entry = 0;
uint64_t ddt_position = 0;
uint64_t secondary_ddt_offset = 0;
uint64_t block_index = 0;
uint8_t *buffer = NULL;
crc64_ctx *crc64_context = NULL;
uint64_t crc64 = 0;
DdtHeader2 ddt_header;
size_t written_bytes = 0;
long end_of_file = 0;
bool create_new_table = false;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Should not really be here
if(ctx->user_data_ddt_header.tableShift == 0)
{
FATAL("DDT table shift is zero, but we are in multi-level DDT setting.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Calculate positive or negative sector
if(negative)
sector_address = ctx->user_data_ddt_header.negative - sector_address;
else
sector_address += ctx->user_data_ddt_header.negative;
// Step 1: Calculate the corresponding secondary level table
items_per_ddt_entry = 1 << ctx->user_data_ddt_header.tableShift;
ddt_position = sector_address / items_per_ddt_entry;
secondary_ddt_offset = ctx->user_data_ddt2[ddt_position];
// Position in file of the child DDT table
secondary_ddt_offset *= 1 << ctx->user_data_ddt_header.blockAlignmentShift;
// Step 2: Check if it corresponds to the currently in-memory cached secondary level table
if(ctx->cached_ddt_offset == secondary_ddt_offset && secondary_ddt_offset != 0)
{
// Update the corresponding DDT entry directly in the cached table
if(*ddt_entry == 0)
{
block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
*ddt_entry = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
block_index << ctx->user_data_ddt_header.dataShift;
// Overflow detection for DDT entry
if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
{
FATAL("DDT overflow: media does not fit in big DDT");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
}
// Sector status can be different from previous deduplicated sector
*ddt_entry &= 0x0FFFFFFFFFFFFFFF;
*ddt_entry |= (uint64_t)sector_status << 60;
TRACE("Setting small secondary DDT entry %d to %ull", sector_address % items_per_ddt_entry,
(uint64_t)*ddt_entry);
ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry] = *ddt_entry;
ctx->dirty_secondary_ddt = true; // Mark secondary DDT as dirty
TRACE("Updated cached secondary DDT entry at position %" PRIu64, sector_address % items_per_ddt_entry);
TRACE("Exiting set_ddt_multi_level_v2() = true");
return true;
}
// Step 2.5: Handle case where we have a cached secondary DDT that has never been written to disk
// but does not contain the requested block
if(ctx->cached_ddt_offset == 0 && (ctx->cached_secondary_ddt2 != NULL))
{
// Only write the cached table to disk if the requested block belongs to a different DDT position
if(ddt_position != ctx->cached_ddt_position)
{
TRACE("Current secondary DDT in memory belongs to position %" PRIu64
" but requested block needs position %" PRIu64,
ctx->cached_ddt_position, ddt_position);
// Write the cached DDT to disk before proceeding with the new one
// Close the current data block first
if(ctx->writing_buffer != NULL) aaruf_close_current_block(ctx);
// Get current position and seek to end of file
fseek(ctx->imageStream, 0, SEEK_END);
end_of_file = ftell(ctx->imageStream);
// Align to block boundary
uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
end_of_file = end_of_file + alignment_mask & ~alignment_mask;
fseek(ctx->imageStream, end_of_file, SEEK_SET);
// Prepare DDT header for the never-written cached table
memset(&ddt_header, 0, sizeof(DdtHeader2));
ddt_header.identifier = DeDuplicationTableSecondary;
ddt_header.type = UserData;
ddt_header.compression = ctx->compression_enabled ? Lzma : None; // Use no compression for simplicity
ddt_header.levels = ctx->user_data_ddt_header.levels;
ddt_header.tableLevel = ctx->user_data_ddt_header.tableLevel + 1;
ddt_header.previousLevelOffset = ctx->primary_ddt_offset;
ddt_header.negative = ctx->user_data_ddt_header.negative;
ddt_header.blocks = items_per_ddt_entry;
ddt_header.overflow = ctx->user_data_ddt_header.overflow;
ddt_header.start = ctx->cached_ddt_position * items_per_ddt_entry; // Use cached position with table shift
ddt_header.blockAlignmentShift = ctx->user_data_ddt_header.blockAlignmentShift;
ddt_header.dataShift = ctx->user_data_ddt_header.dataShift;
ddt_header.tableShift = 0; // Secondary tables are single level
ddt_header.entries = items_per_ddt_entry;
// Calculate data size
ddt_header.length = items_per_ddt_entry * sizeof(uint64_t);
// Calculate CRC64 of the data
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cached_secondary_ddt2, (uint32_t)ddt_header.length);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
ddt_header.crc64 = crc64;
uint8_t *cmp_buffer = NULL;
uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH] = {0};
if(ddt_header.compression == None)
{
cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
ddt_header.cmpCrc64 = ddt_header.crc64;
}
else
{
cmp_buffer = malloc((size_t)ddt_header.length * 2); // Allocate double size for compression
if(cmp_buffer == NULL)
{
TRACE("Failed to allocate memory for secondary DDT v2 compression");
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
size_t dst_size = (size_t)ddt_header.length * 2 * 2;
size_t props_size = LZMA_PROPERTIES_LENGTH;
aaruf_lzma_encode_buffer(cmp_buffer, &dst_size, (uint8_t *)ctx->cached_secondary_ddt2,
ddt_header.length, lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0,
2, 273, 8);
ddt_header.cmpLength = (uint32_t)dst_size;
if(ddt_header.cmpLength >= ddt_header.length)
{
ddt_header.compression = None;
free(cmp_buffer);
cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
}
}
if(ddt_header.compression == None)
{
ddt_header.cmpLength = ddt_header.length;
ddt_header.cmpCrc64 = ddt_header.crc64;
}
else
ddt_header.cmpCrc64 = aaruf_crc64_data(cmp_buffer, (uint32_t)ddt_header.cmpLength);
if(ddt_header.compression == Lzma) ddt_header.cmpLength += LZMA_PROPERTIES_LENGTH;
// Write header
written_bytes = fwrite(&ddt_header, sizeof(DdtHeader2), 1, ctx->imageStream);
if(written_bytes != 1)
{
FATAL("Could not write never-written DDT header to file.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Write data
if(ddt_header.compression == Lzma) fwrite(lzma_properties, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream);
if(fwrite(cmp_buffer, ddt_header.cmpLength, 1, ctx->imageStream) != 1)
{
FATAL("Could not write never-written DDT data to file.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
if(ddt_header.compression == Lzma) free(cmp_buffer);
// Add index entry for the newly written secondary DDT
IndexEntry new_ddt_entry;
new_ddt_entry.blockType = DeDuplicationTableSecondary;
new_ddt_entry.dataType = UserData;
new_ddt_entry.offset = end_of_file;
utarray_push_back(ctx->index_entries, &new_ddt_entry);
ctx->dirty_index_block = true;
TRACE("Added new DDT index entry for never-written table at offset %" PRIu64, end_of_file);
// Update the primary level table entry to point to the new location of the secondary table
uint64_t new_secondary_table_block_offset = end_of_file >> ctx->user_data_ddt_header.blockAlignmentShift;
ctx->user_data_ddt2[ctx->cached_ddt_position] = new_secondary_table_block_offset;
ctx->dirty_primary_ddt = true; // Mark primary DDT as dirty
// Write the updated primary table back to its original position in the file
long saved_pos = ftell(ctx->imageStream);
fseek(ctx->imageStream, ctx->primary_ddt_offset + sizeof(DdtHeader2), SEEK_SET);
size_t primary_table_size = ctx->user_data_ddt_header.entries * sizeof(uint64_t);
written_bytes = fwrite(ctx->user_data_ddt2, primary_table_size, 1, ctx->imageStream);
if(written_bytes != 1)
{
FATAL("Could not flush primary DDT table to file after writing never-written secondary table.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Update nextBlockPosition to ensure future blocks don't overwrite the DDT
uint64_t ddt_total_size = sizeof(DdtHeader2) + ddt_header.length;
ctx->next_block_position = end_of_file + ddt_total_size + alignment_mask & ~alignment_mask;
block_offset = ctx->next_block_position;
offset = 0;
TRACE("Updated nextBlockPosition after never-written DDT write to %" PRIu64, ctx->next_block_position);
// Free the cached table
free(ctx->cached_secondary_ddt2);
ctx->cached_secondary_ddt2 = NULL;
// Reset cached values since we've written and freed the table
ctx->cached_ddt_offset = 0;
ctx->cached_ddt_position = 0;
// Restore file position
fseek(ctx->imageStream, saved_pos, SEEK_SET);
TRACE("Successfully wrote never-written cached secondary DDT to disk");
}
else
// The cached DDT is actually for the requested block range, so we can use it directly
TRACE("Cached DDT is for the correct block range, using it directly");
// No need to write to disk, just continue with the cached table
}
// Step 3: Write the currently in-memory cached secondary level table to the end of the file
if(ctx->cached_ddt_offset != 0)
{
long current_pos = 0;
// Close the current data block first
if(ctx->writing_buffer != NULL) aaruf_close_current_block(ctx);
// Get current position and seek to end of file
current_pos = ftell(ctx->imageStream);
fseek(ctx->imageStream, 0, SEEK_END);
end_of_file = ftell(ctx->imageStream);
// Align to block boundary
uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
end_of_file = end_of_file + alignment_mask & ~alignment_mask;
fseek(ctx->imageStream, end_of_file, SEEK_SET);
// Prepare DDT header for the cached table
memset(&ddt_header, 0, sizeof(DdtHeader2));
ddt_header.identifier = DeDuplicationTableSecondary;
ddt_header.type = UserData;
ddt_header.compression = ctx->compression_enabled ? Lzma : None;
ddt_header.levels = ctx->user_data_ddt_header.levels;
ddt_header.tableLevel = ctx->user_data_ddt_header.tableLevel + 1;
ddt_header.previousLevelOffset = ctx->primary_ddt_offset; // Set to primary DDT table location
ddt_header.negative = ctx->user_data_ddt_header.negative;
ddt_header.blocks = items_per_ddt_entry;
ddt_header.overflow = ctx->user_data_ddt_header.overflow;
ddt_header.start = ddt_position * items_per_ddt_entry; // First block this DDT table references
ddt_header.blockAlignmentShift = ctx->user_data_ddt_header.blockAlignmentShift;
ddt_header.dataShift = ctx->user_data_ddt_header.dataShift;
ddt_header.tableShift = 0; // Secondary tables are single level
ddt_header.entries = items_per_ddt_entry;
// Calculate data size
ddt_header.length = items_per_ddt_entry * sizeof(uint64_t);
// Calculate CRC64 of the data
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cached_secondary_ddt2, ddt_header.length);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
ddt_header.crc64 = crc64;
uint8_t *cmp_buffer = NULL;
uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH] = {0};
if(ddt_header.compression == None)
{
cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
ddt_header.cmpCrc64 = ddt_header.crc64;
}
else
{
cmp_buffer = malloc((size_t)ddt_header.length * 2); // Allocate double size for compression
if(cmp_buffer == NULL)
{
TRACE("Failed to allocate memory for secondary DDT v2 compression");
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
size_t dst_size = (size_t)ddt_header.length * 2 * 2;
size_t props_size = LZMA_PROPERTIES_LENGTH;
aaruf_lzma_encode_buffer(cmp_buffer, &dst_size, (uint8_t *)ctx->cached_secondary_ddt2, ddt_header.length,
lzma_properties, &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
ddt_header.cmpLength = (uint32_t)dst_size;
if(ddt_header.cmpLength >= ddt_header.length)
{
ddt_header.compression = None;
free(cmp_buffer);
cmp_buffer = (uint8_t *)ctx->cached_secondary_ddt2;
}
}
if(ddt_header.compression == None)
{
ddt_header.cmpLength = ddt_header.length;
ddt_header.cmpCrc64 = ddt_header.crc64;
}
else
ddt_header.cmpCrc64 = aaruf_crc64_data(cmp_buffer, (uint32_t)ddt_header.cmpLength);
if(ddt_header.compression == Lzma) ddt_header.cmpLength += LZMA_PROPERTIES_LENGTH;
// Write header
if(ddt_header.compression == Lzma) fwrite(lzma_properties, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream);
written_bytes = fwrite(&ddt_header, sizeof(DdtHeader2), 1, ctx->imageStream);
if(written_bytes != 1)
{
FATAL("Could not write DDT header to file.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Write data
written_bytes = fwrite(cmp_buffer, ddt_header.cmpLength, 1, ctx->imageStream);
if(written_bytes != 1)
{
FATAL("Could not write DDT data to file.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
if(ddt_header.compression == Lzma) free(cmp_buffer);
// Update index: remove old entry and add new one for the evicted secondary DDT
TRACE("Updating index for evicted secondary DDT");
// Remove old index entry for the cached DDT
if(ctx->cached_ddt_offset != 0)
{
TRACE("Removing old index entry for DDT at offset %" PRIu64, ctx->cached_ddt_offset);
IndexEntry *entry = NULL;
// Find and remove the old index entry
for(unsigned int i = 0; i < utarray_len(ctx->index_entries); i++)
{
entry = (IndexEntry *)utarray_eltptr(ctx->index_entries, i);
if(entry && entry->offset == ctx->cached_ddt_offset &&
(entry->blockType == DeDuplicationTableSecondary || entry->blockType == DeDuplicationTableSAlpha))
{
TRACE("Found old DDT index entry at position %u, removing", i);
utarray_erase(ctx->index_entries, i, 1);
break;
}
}
}
// Add new index entry for the newly written secondary DDT
IndexEntry new_ddt_entry;
new_ddt_entry.blockType = DeDuplicationTableSecondary;
new_ddt_entry.dataType = UserData;
new_ddt_entry.offset = end_of_file;
utarray_push_back(ctx->index_entries, &new_ddt_entry);
ctx->dirty_index_block = true;
TRACE("Added new DDT index entry at offset %" PRIu64, end_of_file);
// Step 4: Update the primary level table entry and flush it back to file
uint64_t new_secondary_table_block_offset = end_of_file >> ctx->user_data_ddt_header.blockAlignmentShift;
// Update the primary table entry to point to the new location of the secondary table
// Use ddtPosition which was calculated from sectorAddress, not cachedDdtOffset
ctx->user_data_ddt2[ddt_position] = new_secondary_table_block_offset;
ctx->dirty_primary_ddt = true; // Mark primary DDT as dirty
// Write the updated primary table back to its original position in the file
long saved_pos = ftell(ctx->imageStream);
fseek(ctx->imageStream, ctx->primary_ddt_offset + sizeof(DdtHeader2), SEEK_SET);
size_t primary_table_size = ctx->user_data_ddt_header.entries * sizeof(uint64_t);
written_bytes = fwrite(ctx->user_data_ddt2, primary_table_size, 1, ctx->imageStream);
if(written_bytes != 1)
{
FATAL("Could not flush primary DDT table to file.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Update nextBlockPosition to ensure future blocks don't overwrite the DDT
uint64_t ddt_total_size = sizeof(DdtHeader2) + ddt_header.length;
ctx->next_block_position = end_of_file + ddt_total_size + alignment_mask & ~alignment_mask;
block_offset = ctx->next_block_position;
offset = 0;
TRACE("Updated nextBlockPosition after DDT write to %" PRIu64, ctx->next_block_position);
fseek(ctx->imageStream, saved_pos, SEEK_SET);
// Free the cached table
free(ctx->cached_secondary_ddt2);
ctx->cached_secondary_ddt2 = NULL;
// Restore file position
fseek(ctx->imageStream, current_pos, SEEK_SET);
}
// Step 5: Check if the specified block already has an existing secondary level table
create_new_table = ctx->cached_secondary_ddt2 == NULL;
if(!create_new_table && secondary_ddt_offset != 0)
{
// Load existing table
fseek(ctx->imageStream, secondary_ddt_offset, SEEK_SET);
size_t read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader2), ctx->imageStream);
if(read_bytes != sizeof(DdtHeader2) || ddt_header.identifier != DeDuplicationTable2 ||
ddt_header.type != UserData)
{
FATAL("Invalid secondary DDT header at %" PRIu64, secondary_ddt_offset);
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Read the table data (assuming no compression for now)
buffer = malloc(ddt_header.length);
if(buffer == NULL)
{
FATAL("Cannot allocate memory for secondary DDT.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
read_bytes = fread(buffer, 1, ddt_header.length, ctx->imageStream);
if(read_bytes != ddt_header.length)
{
FATAL("Could not read secondary DDT data.");
free(buffer);
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Verify CRC
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
FATAL("Could not initialize CRC64.");
free(buffer);
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
aaruf_crc64_update(crc64_context, buffer, read_bytes);
aaruf_crc64_final(crc64_context, &crc64);
aaruf_crc64_free(crc64_context);
if(crc64 != ddt_header.crc64)
{
FATAL("Secondary DDT CRC mismatch. Expected 0x%16lX but got 0x%16lX.", ddt_header.crc64, crc64);
free(buffer);
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
// Cache the loaded table
ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
ctx->cached_ddt_offset = secondary_ddt_offset;
}
if(create_new_table)
{
// Create a new empty table
size_t table_size = items_per_ddt_entry * sizeof(uint64_t);
buffer = calloc(1, table_size);
if(buffer == NULL)
{
FATAL("Cannot allocate memory for new secondary DDT.");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
ctx->cached_secondary_ddt2 = (uint64_t *)buffer;
ctx->cached_ddt_offset = 0; // Will be set when written to file
ctx->cached_ddt_position = ddt_position; // Track which primary DDT position this new table belongs to
TRACE("Created new secondary DDT for position %" PRIu64, ddt_position);
}
// Step 6: Update the corresponding DDT entry
if(*ddt_entry == 0)
{
block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
*ddt_entry = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
block_index << ctx->user_data_ddt_header.dataShift;
// Overflow detection for DDT entry
if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
{
FATAL("DDT overflow: media does not fit in big DDT");
TRACE("Exiting set_ddt_multi_level_v2() = false");
return false;
}
}
// Sector status can be different from previous deduplicated sector
*ddt_entry &= 0x0FFFFFFFFFFFFFFF;
*ddt_entry |= (uint64_t)sector_status << 60;
TRACE("Setting big secondary DDT entry %d to %ull", sector_address % items_per_ddt_entry, (uint64_t)*ddt_entry);
ctx->cached_secondary_ddt2[sector_address % items_per_ddt_entry] = *ddt_entry;
ctx->dirty_secondary_ddt = true;
TRACE("Updated secondary DDT entry at position %" PRIu64, sector_address % items_per_ddt_entry);
TRACE("Exiting set_ddt_multi_level_v2() = true");
return true;
}
bool set_ddt_tape(aaruformat_context *ctx, uint64_t sector_address, const uint64_t offset, const uint64_t block_offset,
const uint8_t sector_status, uint64_t *ddt_entry)
{
TRACE("Entering set_ddt_tape(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sector_address, offset, block_offset,
sector_status);
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context or image stream.");
TRACE("Exiting set_ddt_tape() = false");
return false;
}
// Should not really be here
if(!ctx->is_tape)
{
FATAL("Image is not tape, wrong function called.");
TRACE("Exiting set_ddt_tape() = false");
return false;
}
if(*ddt_entry == 0)
{
const uint64_t block_index = block_offset >> ctx->user_data_ddt_header.blockAlignmentShift;
*ddt_entry = offset & (1ULL << ctx->user_data_ddt_header.dataShift) - 1 |
block_index << ctx->user_data_ddt_header.dataShift;
// Overflow detection for DDT entry
if(*ddt_entry > 0xFFFFFFFFFFFFFFF)
{
FATAL("DDT overflow: media does not fit in big DDT");
TRACE("Exiting set_ddt_tape() = false");
return false;
}
*ddt_entry |= (uint64_t)sector_status << 60;
}
// Create DDT hash entry
TapeDdtHashEntry *new_entry = calloc(1, sizeof(TapeDdtHashEntry));
TapeDdtHashEntry *old_entry = NULL;
if(new_entry == NULL)
{
FATAL("Cannot allocate memory for new tape DDT hash entry.");
TRACE("Exiting set_ddt_tape() = false");
return false;
}
TRACE("Setting tape DDT entry %d to %u", sector_address, (uint32_t)*ddt_entry);
new_entry->key = sector_address;
new_entry->value = *ddt_entry;
// Insert entry into tape DDT
HASH_REPLACE(hh, ctx->tape_ddt, key, sizeof(uint64_t), new_entry, old_entry);
ctx->dirty_tape_ddt = true; // Mark tape DDT as dirty
if(old_entry) free(old_entry);
TRACE("Exiting set_ddt_tape() = true");
return true;
}
aaruformat.h
LZMA_PROPERTIES_LENGTH
#define LZMA_PROPERTIES_LENGTH
Size in bytes of the fixed LZMA properties header (lc/lp/pb + dictionary size).
Definition consts.h:82
process_ddt_v2
int32_t process_ddt_v2(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
Processes a DDT v2 block from the image stream.
Definition ddt_v2.c:96
set_ddt_tape
bool set_ddt_tape(aaruformat_context *ctx, uint64_t sector_address, const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status, uint64_t *ddt_entry)
Sets a DDT entry for tape media using a hash-based lookup table.
Definition ddt_v2.c:1800
decode_ddt_single_level_v2
int32_t decode_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status)
Decodes a single-level DDT v2 entry for a given sector address.
Definition ddt_v2.c:585
set_ddt_multi_level_v2
bool set_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t offset, uint64_t block_offset, uint8_t sector_status, uint64_t *ddt_entry)
Sets a multi-level DDT v2 entry for a given sector address.
Definition ddt_v2.c:1110
set_ddt_entry_v2
bool set_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, const bool negative, const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status, uint64_t *ddt_entry)
Sets a DDT v2 entry for a given sector address.
Definition ddt_v2.c:1002
decode_ddt_multi_level_v2
int32_t decode_ddt_multi_level_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status)
Decodes a multi-level DDT v2 entry for a given sector address.
Definition ddt_v2.c:728
set_ddt_single_level_v2
bool set_ddt_single_level_v2(aaruformat_context *ctx, uint64_t sector_address, const bool negative, const uint64_t offset, const uint64_t block_offset, const uint8_t sector_status, uint64_t *ddt_entry)
Sets a single-level DDT v2 entry for a given sector address.
Definition ddt_v2.c:1038
decode_ddt_entry_v2
int32_t decode_ddt_entry_v2(aaruformat_context *ctx, const uint64_t sector_address, bool negative, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status)
Decodes a DDT v2 entry for a given sector address.
Definition ddt_v2.c:511
aaruf_lzma_encode_buffer
int32_t aaruf_lzma_encode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t src_size, uint8_t *out_props, size_t *out_props_size, int32_t level, uint32_t dict_size, int32_t lc, int32_t lp, int32_t pb, int32_t fb, int32_t num_threads)
Encodes a buffer using LZMA compression.
Definition lzma.c:65
aaruf_crc64_data
uint64_t aaruf_crc64_data(const uint8_t *data, uint32_t len)
Definition crc64.c:160
aaruf_lzma_decode_buffer
int32_t aaruf_lzma_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t *src_size, const uint8_t *props, size_t props_size)
Decodes an LZMA-compressed buffer.
Definition lzma.c:39
aaruf_crc64_update
int aaruf_crc64_update(crc64_ctx *ctx, const uint8_t *data, uint32_t len)
Updates the CRC64 context with new data.
Definition crc64.c:55
aaruf_crc64_free
void aaruf_crc64_free(crc64_ctx *ctx)
Frees a CRC64 context.
Definition crc64.c:155
aaruf_crc64_init
crc64_ctx * aaruf_crc64_init()
Initializes a CRC64 context.
Definition crc64.c:32
aaruf_crc64_final
int aaruf_crc64_final(crc64_ctx *ctx, uint64_t *crc)
Computes the final CRC64 value from the context.
Definition crc64.c:141
DeDuplicationTableSecondary
@ DeDuplicationTableSecondary
Block containing a secondary deduplication table (v2).
Definition enums.h:148
DeDuplicationTable2
@ DeDuplicationTable2
Block containing a deduplication table v2.
Definition enums.h:146
DeDuplicationTableSAlpha
@ DeDuplicationTableSAlpha
Block containing a secondary deduplication table (v2) (mistake).
Definition enums.h:147
SectorStatusNotDumped
@ SectorStatusNotDumped
Sector(s) not yet acquired during image dumping.
Definition enums.h:236
CdSectorPrefix
@ CdSectorPrefix
Compact Disc sector prefix (sync, header).
Definition enums.h:114
UserData
@ UserData
User (main) data.
Definition enums.h:46
CdSectorSuffix
@ CdSectorSuffix
Compact Disc sector suffix (EDC, ECC P, ECC Q).
Definition enums.h:115
Lzma
@ Lzma
LZMA compression.
Definition enums.h:34
None
@ None
Not compressed.
Definition enums.h:33
AARUF_STATUS_OK
#define AARUF_STATUS_OK
Sector present and read without uncorrectable errors.
Definition errors.h:77
AARUF_ERROR_NOT_ENOUGH_MEMORY
#define AARUF_ERROR_NOT_ENOUGH_MEMORY
Memory allocation failure (critical).
Definition errors.h:48
AARUF_ERROR_CANNOT_READ_BLOCK
#define AARUF_ERROR_CANNOT_READ_BLOCK
Generic block read failure (seek/read error).
Definition errors.h:46
AARUF_ERROR_INVALID_BLOCK_CRC
#define AARUF_ERROR_INVALID_BLOCK_CRC
CRC64 mismatch indicating corruption.
Definition errors.h:57
AARUF_ERROR_NOT_AARUFORMAT
#define AARUF_ERROR_NOT_AARUFORMAT
Input file/stream failed magic or structural validation.
Definition errors.h:40
AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
#define AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
Decompression routine failed or size mismatch.
Definition errors.h:56
internal.h
aaruf_close_current_block
int32_t aaruf_close_current_block(aaruformat_context *ctx)
Finalizes and writes the current data block to the AaruFormat image file.
Definition write.c:1438
log.h
FATAL
#define FATAL(fmt,...)
Definition log.h:40
TRACE
#define TRACE(fmt,...)
Definition log.h:25
DdtHeader2
Header preceding a version 2 hierarchical deduplication table.
Definition ddt.h:142
DdtHeader2::cmpCrc64
uint64_t cmpCrc64
CRC64-ECMA of compressed table payload.
Definition ddt.h:161
DdtHeader2::type
uint16_t type
Data classification (DataType) for sectors referenced by this table.
Definition ddt.h:144
DdtHeader2::start
uint64_t start
Base internal index covered by this table (used for secondary tables; currently informational).
Definition ddt.h:153
DdtHeader2::crc64
uint64_t crc64
CRC64-ECMA of uncompressed table payload.
Definition ddt.h:162
DdtHeader2::entries
uint64_t entries
Number of entries contained in (uncompressed) table payload.
Definition ddt.h:158
DdtHeader2::levels
uint8_t levels
Total number of hierarchy levels (root depth); > 0.
Definition ddt.h:146
DdtHeader2::length
uint64_t length
Uncompressed payload size in bytes.
Definition ddt.h:160
DdtHeader2::identifier
uint32_t identifier
Block identifier, must be BlockType::DeDuplicationTable2.
Definition ddt.h:143
DdtHeader2::tableShift
uint8_t tableShift
2^tableShift = number of logical sectors per primary entry (multi-level only; 0 for single-level or s...
Definition ddt.h:156
DdtHeader2::blocks
uint64_t blocks
Total internal span (negative + usable + overflow) in logical sectors.
Definition ddt.h:150
DdtHeader2::blockAlignmentShift
uint8_t blockAlignmentShift
2^blockAlignmentShift = block alignment boundary in bytes.
Definition ddt.h:154
DdtHeader2::overflow
uint32_t overflow
Trailing dumped sectors beyond user area (overflow range), still mapped with entries.
Definition ddt.h:151
DdtHeader2::negative
uint32_t negative
Leading negative LBA count; added to external L to build internal index.
Definition ddt.h:149
DdtHeader2::tableLevel
uint8_t tableLevel
Zero-based level index of this table (0 = root, increases downward).
Definition ddt.h:147
DdtHeader2::compression
uint16_t compression
Compression algorithm for this table body (CompressionType).
Definition ddt.h:145
DdtHeader2::dataShift
uint8_t dataShift
2^dataShift = sectors represented per increment in blockIndex field.
Definition ddt.h:155
DdtHeader2::cmpLength
uint64_t cmpLength
Compressed payload size in bytes.
Definition ddt.h:159
DdtHeader2::previousLevelOffset
uint64_t previousLevelOffset
Absolute byte offset of the parent (previous) level table; 0 if root.
Definition ddt.h:148
ImageInfo::ImageSize
uint64_t ImageSize
Size of the image payload in bytes (excludes headers/metadata).
Definition aaru.h:935
ImageInfo::Sectors
uint64_t Sectors
Total count of addressable logical sectors/blocks.
Definition aaru.h:936
IndexEntry
Single index entry describing a block's type, (optional) data classification, and file offset.
Definition index.h:109
IndexEntry::blockType
uint32_t blockType
Block identifier of the referenced block (value from BlockType).
Definition index.h:110
IndexEntry::offset
uint64_t offset
Absolute byte offset in the image where the referenced block header begins.
Definition index.h:112
IndexEntry::dataType
uint16_t dataType
Data classification (value from DataType) or unused for untyped blocks.
Definition index.h:111
TapeDdtHashEntry
Definition context.h:144
TapeDdtHashEntry::key
uint64_t key
Key: sector address.
Definition context.h:145
TapeDdtHashEntry::value
uint64_t value
Value: DDT entry.
Definition context.h:146
aaruformat_context
Master context representing an open or in‑creation Aaru image.
Definition context.h:175
aaruformat_context::user_data_ddt_header
DdtHeader2 user_data_ddt_header
Active user data DDT v2 header (primary table meta).
Definition context.h:192
aaruformat_context::compression_enabled
bool compression_enabled
True if block compression enabled (writing path).
Definition context.h:303
aaruformat_context::dirty_primary_ddt
bool dirty_primary_ddt
True if primary DDT table should be written during close.
Definition context.h:318
aaruformat_context::user_data_ddt2
uint64_t * user_data_ddt2
DDT entries (big variant) primary/secondary current.
Definition context.h:190
aaruformat_context::sector_suffix_ddt2
uint64_t * sector_suffix_ddt2
CD sector suffix DDT V2.
Definition context.h:189
aaruformat_context::cached_ddt_offset
uint64_t cached_ddt_offset
File offset of currently cached secondary DDT (0=none).
Definition context.h:193
aaruformat_context::is_tape
bool is_tape
True if the image is a tape image.
Definition context.h:308
aaruformat_context::in_memory_ddt
bool in_memory_ddt
True if primary (and possibly secondary) DDT loaded.
Definition context.h:199
aaruformat_context::dirty_index_block
bool dirty_index_block
True if index block should be written during close.
Definition context.h:340
aaruformat_context::tape_ddt
TapeDdtHashEntry * tape_ddt
Hash table root for tape DDT entries.
Definition context.h:185
aaruformat_context::dirty_single_level_ddt
bool dirty_single_level_ddt
True if single-level DDT should be written during close.
Definition context.h:319
aaruformat_context::ddt_version
int ddt_version
DDT version in use (1=legacy, 2=v2 hierarchical).
Definition context.h:197
aaruformat_context::writing_buffer
uint8_t * writing_buffer
Accumulation buffer for current block data.
Definition context.h:283
aaruformat_context::sector_prefix_ddt2
uint64_t * sector_prefix_ddt2
CD sector prefix DDT V2.
Definition context.h:188
aaruformat_context::primary_ddt_offset
uint64_t primary_ddt_offset
File offset of the primary DDT v2 table.
Definition context.h:195
aaruformat_context::next_block_position
uint64_t next_block_position
Absolute file offset where next block will be written.
Definition context.h:285
aaruformat_context::cached_secondary_ddt2
uint64_t * cached_secondary_ddt2
Cached secondary table (big entries) or NULL.
Definition context.h:191
aaruformat_context::imageStream
FILE * imageStream
Underlying FILE* stream (binary mode).
Definition context.h:179
aaruformat_context::index_entries
UT_array * index_entries
Flattened index entries (UT_array of IndexEntry).
Definition context.h:255
aaruformat_context::image_info
ImageInfo image_info
Exposed high-level image info summary.
Definition context.h:263
aaruformat_context::dirty_secondary_ddt
bool dirty_secondary_ddt
True if secondary DDT tables should be written during close.
Definition context.h:317
aaruformat_context::dirty_tape_ddt
bool dirty_tape_ddt
True if tape DDT should be written during close.
Definition context.h:331
aaruformat_context::lzma_dict_size
uint32_t lzma_dict_size
LZMA dictionary size (writing path).
Definition context.h:301
aaruformat_context::cached_ddt_position
uint64_t cached_ddt_position
Position index of cached secondary DDT.
Definition context.h:194
crc64_ctx
Minimal ECMA-182 CRC64 incremental state container (running value only).
Definition crc64.h:56