From d8e39eb87be34bc89d0da0dd3875694c762f2c69 Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sun, 28 Sep 2025 15:15:05 +0100 Subject: [PATCH] Add primary DDT offset and implement DDT entry setting functions --- include/aaruformat/context.h | 7 +- include/internal.h | 4 + src/ddt/ddt_v2.c | 367 ++++++++++++++++++++++++++++++++++- 3 files changed, 373 insertions(+), 5 deletions(-) diff --git a/include/aaruformat/context.h b/include/aaruformat/context.h index 6613ad4..6077494 100644 --- a/include/aaruformat/context.h +++ b/include/aaruformat/context.h @@ -98,15 +98,15 @@ typedef struct aaruformatContext uint8_t *cicmBlock; DumpHardwareHeader dumpHardwareHeader; struct DumpHardwareEntriesWithData *dumpHardwareEntriesWithData; - struct ImageInfo imageInfo; + ImageInfo imageInfo; CdEccContext *eccCdContext; uint8_t numberOfDataTracks; TrackEntry *dataTracks; bool *readableSectorTags; struct CacheHeader blockHeaderCache; struct CacheHeader blockCache; - struct Checksums checksums; - struct mediaTagEntry *mediaTags; + Checksums checksums; + mediaTagEntry *mediaTags; DdtHeader2 userDataDdtHeader; int ddtVersion; uint16_t *userDataDdtMini; @@ -114,6 +114,7 @@ typedef struct aaruformatContext uint16_t *sectorPrefixDdtMini; uint16_t *sectorSuffixDdtMini; uint64_t cachedDdtOffset; + uint64_t primaryDdtOffset; uint16_t *cachedSecondaryDdtSmall; uint32_t *cachedSecondaryDdtBig; bool isWriting; diff --git a/include/internal.h b/include/internal.h index c253013..dc0ea3e 100644 --- a/include/internal.h +++ b/include/internal.h @@ -45,6 +45,10 @@ int32_t decode_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddr uint64_t *blockOffset, uint8_t *sectorStatus); int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, uint64_t *offset, uint64_t *blockOffset, uint8_t *sectorStatus); +void set_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset, + uint64_t blockOffset, uint8_t sectorStatus); +void set_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset, + uint64_t blockOffset, uint8_t sectorStatus); aaru_options parse_options(const char *options); uint64_t get_filetime_uint64(); int32_t aaruf_close_current_block(aaruformatContext *ctx); diff --git a/src/ddt/ddt_v2.c b/src/ddt/ddt_v2.c index 15ddf7c..76364cc 100644 --- a/src/ddt/ddt_v2.c +++ b/src/ddt/ddt_v2.c @@ -82,6 +82,8 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse // We need the header later for the shift calculations ctx->userDataDdtHeader = ddtHeader; ctx->ddtVersion = 2; + // Store the primary DDT table's file offset for secondary table references + ctx->primaryDdtOffset = entry->offset; // Check for DDT compression switch(ddtHeader.compression) @@ -358,7 +360,7 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse if(crc64_context == NULL) { - FATAL(stderr, "Could not initialize CRC64."); + FATAL("Could not initialize CRC64."); free(buffer); TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK"); return AARUF_ERROR_CANNOT_READ_BLOCK; @@ -371,6 +373,7 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse { FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddtHeader.crc64, crc64); free(buffer); + TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC"); return AARUF_ERROR_INVALID_BLOCK_CRC; } @@ -544,7 +547,7 @@ int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress } // Position in file of the child DDT table - secondaryDdtOffset *= (1 << ctx->userDataDdtHeader.blockAlignmentShift); + secondaryDdtOffset *= 1 << ctx->userDataDdtHeader.blockAlignmentShift; // Is the one we have cached the same as the one we need to read? if(ctx->cachedDdtOffset != secondaryDdtOffset) @@ -753,4 +756,364 @@ int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx, sectorAddress, *offset, *blockOffset, *sectorStatus); return AARUF_STATUS_OK; +} + +void set_ddt_entry_v2(aaruformatContext *ctx, uint64_t sectorAddress, uint64_t offset, uint64_t blockOffset, + uint8_t sectorStatus) +{ + TRACE("Entering set_ddt_entry_v2(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sectorAddress, offset, blockOffset, + sectorStatus); + + // Check if the context and image stream are valid + if(ctx == NULL || ctx->imageStream == NULL) + { + FATAL("Invalid context or image stream."); + return; + } + + if(ctx->userDataDdtHeader.tableShift > 0) + set_ddt_multi_level_v2(ctx, sectorAddress, false, offset, blockOffset, sectorStatus); + else + set_ddt_single_level_v2(ctx, sectorAddress, false, offset, blockOffset, sectorStatus); +} + +void set_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset, + uint64_t blockOffset, uint8_t sectorStatus) +{ + TRACE("Entering set_ddt_single_level_v2(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sectorAddress, offset, blockOffset, + sectorStatus); + + // Check if the context and image stream are valid + if(ctx == NULL || ctx->imageStream == NULL) + { + FATAL("Invalid context or image stream."); + return; + } + + // Should not really be here + if(ctx->userDataDdtHeader.tableShift != 0) + { + FATAL("DDT table shift is not zero, but we are in single-level DDT setting."); + return; + } + + // Calculate positive or negative sector + if(negative) + sectorAddress -= ctx->userDataDdtHeader.negative; + else + sectorAddress += ctx->userDataDdtHeader.negative; + + uint64_t ddtEntry = 0; + + uint64_t blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift; + ddtEntry = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex << ctx->userDataDdtHeader.dataShift; + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 12; + ctx->cachedSecondaryDdtSmall[sectorAddress] = (uint16_t)ddtEntry; + } + else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 28; + ctx->cachedSecondaryDdtBig[sectorAddress] = (uint32_t)ddtEntry; + } +} + +void set_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset, + uint64_t blockOffset, uint8_t sectorStatus) +{ + TRACE("Entering set_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %" PRIu64 ", %" PRIu64 ", %d)", ctx, sectorAddress, + negative, offset, blockOffset, sectorStatus); + + uint64_t itemsPerDdtEntry = 0; + uint64_t ddtPosition = 0; + uint64_t secondaryDdtOffset = 0; + uint64_t ddtEntry = 0; + uint64_t blockIndex = 0; + uint8_t *buffer = NULL; + crc64_ctx *crc64_context = NULL; + uint64_t crc64 = 0; + DdtHeader2 ddtHeader; + size_t writtenBytes = 0; + long currentPos = 0; + long endOfFile = 0; + bool createNewTable = false; + + // Check if the context and image stream are valid + if(ctx == NULL || ctx->imageStream == NULL) + { + FATAL("Invalid context or image stream."); + return; + } + + // Should not really be here + if(ctx->userDataDdtHeader.tableShift == 0) + { + FATAL("DDT table shift is zero, but we are in multi-level DDT setting."); + return; + } + + // Calculate positive or negative sector + if(negative) + sectorAddress -= ctx->userDataDdtHeader.negative; + else + sectorAddress += ctx->userDataDdtHeader.negative; + + // Step 1: Calculate the corresponding secondary level table + itemsPerDdtEntry = 1 << ctx->userDataDdtHeader.tableShift; + ddtPosition = sectorAddress / itemsPerDdtEntry; + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + secondaryDdtOffset = ctx->userDataDdtMini[ddtPosition]; + else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType) + secondaryDdtOffset = ctx->userDataDdtBig[ddtPosition]; + else + { + FATAL("Unknown DDT size type %d.", ctx->userDataDdtHeader.sizeType); + return; + } + + // Position in file of the child DDT table + secondaryDdtOffset *= 1 << ctx->userDataDdtHeader.blockAlignmentShift; + + // Step 2: Check if it corresponds to the currently in-memory cached secondary level table + if(ctx->cachedDdtOffset == secondaryDdtOffset && secondaryDdtOffset != 0) + { + // Update the corresponding DDT entry directly in the cached table + blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift; + ddtEntry = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex + << ctx->userDataDdtHeader.dataShift; + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 12; + ctx->cachedSecondaryDdtSmall[sectorAddress % itemsPerDdtEntry] = (uint16_t)ddtEntry; + } + else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 28; + ctx->cachedSecondaryDdtBig[sectorAddress % itemsPerDdtEntry] = (uint32_t)ddtEntry; + } + + TRACE("Updated cached secondary DDT entry at position %" PRIu64, sectorAddress % itemsPerDdtEntry); + return; + } + + // Step 3: Write the currently in-memory cached secondary level table to the end of the file + if(ctx->cachedDdtOffset != 0) + { + // Get current position and seek to end of file + currentPos = ftell(ctx->imageStream); + fseek(ctx->imageStream, 0, SEEK_END); + endOfFile = ftell(ctx->imageStream); + + // Prepare DDT header for the cached table + memset(&ddtHeader, 0, sizeof(DdtHeader2)); + ddtHeader.identifier = DeDuplicationTable2; + ddtHeader.type = UserData; + ddtHeader.compression = None; // Use no compression for simplicity + ddtHeader.levels = ctx->userDataDdtHeader.levels; + ddtHeader.tableLevel = ctx->userDataDdtHeader.tableLevel + 1; + ddtHeader.previousLevelOffset = ctx->primaryDdtOffset; // Set to primary DDT table location + ddtHeader.negative = ctx->userDataDdtHeader.negative; + ddtHeader.blocks = itemsPerDdtEntry; + ddtHeader.overflow = ctx->userDataDdtHeader.overflow; + ddtHeader.start = ddtPosition * itemsPerDdtEntry; // First block this DDT table references + ddtHeader.blockAlignmentShift = ctx->userDataDdtHeader.blockAlignmentShift; + ddtHeader.dataShift = ctx->userDataDdtHeader.dataShift; + ddtHeader.tableShift = 0; // Secondary tables are single level + ddtHeader.sizeType = ctx->userDataDdtHeader.sizeType; + ddtHeader.entries = itemsPerDdtEntry; + + // Calculate data size + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + ddtHeader.length = itemsPerDdtEntry * sizeof(uint16_t); + else + ddtHeader.length = itemsPerDdtEntry * sizeof(uint32_t); + + ddtHeader.cmpLength = ddtHeader.length; + + // Calculate CRC64 of the data + crc64_context = aaruf_crc64_init(); + if(crc64_context == NULL) + { + FATAL("Could not initialize CRC64."); + return; + } + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cachedSecondaryDdtSmall, ddtHeader.length); + else + aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cachedSecondaryDdtBig, ddtHeader.length); + + aaruf_crc64_final(crc64_context, &crc64); + ddtHeader.crc64 = crc64; + ddtHeader.cmpCrc64 = crc64; + + // Write header + writtenBytes = fwrite(&ddtHeader, sizeof(DdtHeader2), 1, ctx->imageStream); + if(writtenBytes != 1) + { + FATAL("Could not write DDT header to file."); + return; + } + + // Write data + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + writtenBytes = fwrite(ctx->cachedSecondaryDdtSmall, ddtHeader.length, 1, ctx->imageStream); + else + writtenBytes = fwrite(ctx->cachedSecondaryDdtBig, ddtHeader.length, 1, ctx->imageStream); + + if(writtenBytes != 1) + { + FATAL("Could not write DDT data to file."); + return; + } + + // Step 4: Update the primary level table entry and flush it back to file + uint64_t newSecondaryTableBlockOffset = endOfFile >> ctx->userDataDdtHeader.blockAlignmentShift; + + // Find which entry in the primary table corresponds to the cached secondary table + uint64_t cachedDdtPosition = ctx->cachedDdtOffset >> ctx->userDataDdtHeader.blockAlignmentShift; + + // Update the primary table entry to point to the new location of the secondary table + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + ctx->userDataDdtMini[cachedDdtPosition] = (uint16_t)newSecondaryTableBlockOffset; + else + ctx->userDataDdtBig[cachedDdtPosition] = (uint32_t)newSecondaryTableBlockOffset; + + // Write the updated primary table back to its original position in the file + long savedPos = ftell(ctx->imageStream); + fseek(ctx->imageStream, ctx->primaryDdtOffset + sizeof(DdtHeader2), SEEK_SET); + + size_t primaryTableSize = ctx->userDataDdtHeader.sizeType == SmallDdtSizeType + ? ctx->userDataDdtHeader.entries * sizeof(uint16_t) + : ctx->userDataDdtHeader.entries * sizeof(uint32_t); + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + writtenBytes = fwrite(ctx->userDataDdtMini, primaryTableSize, 1, ctx->imageStream); + else + writtenBytes = fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream); + + if(writtenBytes != 1) + { + FATAL("Could not flush primary DDT table to file."); + return; + } + + fseek(ctx->imageStream, savedPos, SEEK_SET); + + // Free the cached table + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType && ctx->cachedSecondaryDdtSmall) + { + free(ctx->cachedSecondaryDdtSmall); + ctx->cachedSecondaryDdtSmall = NULL; + } + else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType && ctx->cachedSecondaryDdtBig) + { + free(ctx->cachedSecondaryDdtBig); + ctx->cachedSecondaryDdtBig = NULL; + } + + // Restore file position + fseek(ctx->imageStream, currentPos, SEEK_SET); + } + + // Step 5: Check if the specified block already has an existing secondary level table + createNewTable = secondaryDdtOffset == 0; + + if(!createNewTable) + { + // Load existing table + fseek(ctx->imageStream, secondaryDdtOffset, SEEK_SET); + size_t readBytes = fread(&ddtHeader, 1, sizeof(DdtHeader2), ctx->imageStream); + + if(readBytes != sizeof(DdtHeader2) || ddtHeader.identifier != DeDuplicationTable2 || ddtHeader.type != UserData) + { + FATAL("Invalid secondary DDT header at %" PRIu64, secondaryDdtOffset); + return; + } + + // Read the table data (assuming no compression for now) + buffer = malloc(ddtHeader.length); + if(buffer == NULL) + { + FATAL("Cannot allocate memory for secondary DDT."); + return; + } + + readBytes = fread(buffer, 1, ddtHeader.length, ctx->imageStream); + if(readBytes != ddtHeader.length) + { + FATAL("Could not read secondary DDT data."); + free(buffer); + return; + } + + // Verify CRC + crc64_context = aaruf_crc64_init(); + if(crc64_context == NULL) + { + FATAL("Could not initialize CRC64."); + free(buffer); + return; + } + + aaruf_crc64_update(crc64_context, buffer, readBytes); + aaruf_crc64_final(crc64_context, &crc64); + + if(crc64 != ddtHeader.crc64) + { + FATAL("Secondary DDT CRC mismatch. Expected 0x%16lX but got 0x%16lX.", ddtHeader.crc64, crc64); + free(buffer); + return; + } + + // Cache the loaded table + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + ctx->cachedSecondaryDdtSmall = (uint16_t *)buffer; + else + ctx->cachedSecondaryDdtBig = (uint32_t *)buffer; + + ctx->cachedDdtOffset = secondaryDdtOffset; + } + else + { + // Create a new empty table + size_t tableSize = ctx->userDataDdtHeader.sizeType == SmallDdtSizeType ? itemsPerDdtEntry * sizeof(uint16_t) + : itemsPerDdtEntry * sizeof(uint32_t); + + buffer = calloc(1, tableSize); + if(buffer == NULL) + { + FATAL("Cannot allocate memory for new secondary DDT."); + return; + } + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + ctx->cachedSecondaryDdtSmall = (uint16_t *)buffer; + else + ctx->cachedSecondaryDdtBig = (uint32_t *)buffer; + + ctx->cachedDdtOffset = 0; // Will be set when written to file + } + + // Step 6: Update the corresponding DDT entry + blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift; + ddtEntry = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex << ctx->userDataDdtHeader.dataShift; + + if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 12; + ctx->cachedSecondaryDdtSmall[sectorAddress % itemsPerDdtEntry] = (uint16_t)ddtEntry; + } + else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType) + { + ddtEntry |= (uint64_t)sectorStatus << 28; + ctx->cachedSecondaryDdtBig[sectorAddress % itemsPerDdtEntry] = (uint32_t)ddtEntry; + } + + TRACE("Updated secondary DDT entry at position %" PRIu64, sectorAddress % itemsPerDdtEntry); + TRACE("Exiting set_ddt_multi_level_v2()"); } \ No newline at end of file