From 90ca1ee872203828b86a7fb16a84bb0ee5796c7e Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sun, 28 Sep 2025 17:08:33 +0100 Subject: [PATCH] Initialize index entries array and update index management for DDT entries --- include/aaruformat/context.h | 2 + src/close.c | 176 ++++++++++++++++++++++++++++++++++- src/create.c | 16 ++++ src/ddt/ddt_v2.c | 33 ++++++- src/write.c | 10 +- 5 files changed, 230 insertions(+), 7 deletions(-) diff --git a/include/aaruformat/context.h b/include/aaruformat/context.h index 6077494..80fdb6d 100644 --- a/include/aaruformat/context.h +++ b/include/aaruformat/context.h @@ -22,6 +22,7 @@ #include "crc64.h" #include "lru.h" #include "structs.h" +#include "utarray.h" #ifndef MD5_DIGEST_LENGTH #define MD5_DIGEST_LENGTH 16 @@ -124,6 +125,7 @@ typedef struct aaruformatContext crc64_ctx *crc64Context; int writingBufferPosition; long nextBlockPosition; + UT_array *indexEntries; } aaruformatContext; typedef struct DumpHardwareEntriesWithData diff --git a/src/close.c b/src/close.c index 3e3d4c1..fb6624a 100644 --- a/src/close.c +++ b/src/close.c @@ -171,19 +171,59 @@ int aaruf_close(void *context) else ctx->userDataDdtBig[cachedDdtPosition] = (uint32_t)newSecondaryTableBlockOffset; - // Write updated primary table back to its original position + // Update index: remove old entry for cached DDT and add new one + TRACE("Updating index for cached secondary DDT"); + + // Remove old index entry for the cached DDT + if(ctx->cachedDdtOffset != 0) + { + TRACE("Removing old index entry for DDT at offset %" PRIu64, ctx->cachedDdtOffset); + IndexEntry *entry = NULL; + + // Find and remove the old index entry + for(unsigned int i = 0; i < utarray_len(ctx->indexEntries); i++) + { + entry = (IndexEntry *)utarray_eltptr(ctx->indexEntries, i); + if(entry && entry->offset == ctx->cachedDdtOffset && + entry->blockType == DeDuplicationTable2) + { + TRACE("Found old DDT index entry at position %u, removing", i); + utarray_erase(ctx->indexEntries, i, 1); + break; + } + } + } + + // Add new index entry for the newly written secondary DDT + IndexEntry newDdtEntry; + newDdtEntry.blockType = DeDuplicationTable2; + newDdtEntry.dataType = UserData; + newDdtEntry.offset = endOfFile; + + utarray_push_back(ctx->indexEntries, &newDdtEntry); + TRACE("Added new DDT index entry at offset %" PRIu64, endOfFile); + + // Write the updated primary table back to its original position in the file + long savedPos = ftell(ctx->imageStream); fseek(ctx->imageStream, ctx->primaryDdtOffset + sizeof(DdtHeader2), SEEK_SET); size_t primaryTableSize = ctx->userDataDdtHeader.sizeType == SmallDdtSizeType ? ctx->userDataDdtHeader.entries * sizeof(uint16_t) : ctx->userDataDdtHeader.entries * sizeof(uint32_t); + size_t primaryWrittenBytes = 0; if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType) - fwrite(ctx->userDataDdtMini, primaryTableSize, 1, ctx->imageStream); + primaryWrittenBytes = fwrite(ctx->userDataDdtMini, primaryTableSize, 1, ctx->imageStream); else - fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream); + primaryWrittenBytes = fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream); - TRACE("Successfully wrote cached secondary DDT table and updated primary table"); + if(primaryWrittenBytes != 1) + { + TRACE("Could not flush primary DDT table to file."); + return AARUF_ERROR_CANNOT_WRITE_HEADER; + } + + fseek(ctx->imageStream, savedPos, SEEK_SET); } else TRACE("Failed to write cached secondary DDT data"); @@ -266,8 +306,20 @@ int aaruf_close(void *context) writtenBytes = fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream); if(writtenBytes == 1) + { TRACE("Successfully wrote primary DDT header and table to file (%" PRIu64 " entries, %zu bytes)", ctx->userDataDdtHeader.entries, primaryTableSize); + + // Add primary DDT to index + TRACE("Adding primary DDT to index"); + IndexEntry primaryDdtEntry; + primaryDdtEntry.blockType = DeDuplicationTable2; + primaryDdtEntry.dataType = UserData; + primaryDdtEntry.offset = ctx->primaryDdtOffset; + + utarray_push_back(ctx->indexEntries, &primaryDdtEntry); + TRACE("Added primary DDT index entry at offset %" PRIu64, ctx->primaryDdtOffset); + } else TRACE("Failed to write primary DDT table to file"); } @@ -333,11 +385,118 @@ int aaruf_close(void *context) writtenBytes = fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream); if(writtenBytes == 1) + { TRACE("Successfully wrote single-level DDT header and table to file (%" PRIu64 " entries, %zu bytes)", ctx->userDataDdtHeader.entries, primaryTableSize); + + // Add single-level DDT to index + TRACE("Adding single-level DDT to index"); + IndexEntry singleDdtEntry; + singleDdtEntry.blockType = DeDuplicationTable2; + singleDdtEntry.dataType = UserData; + singleDdtEntry.offset = ctx->primaryDdtOffset; + + utarray_push_back(ctx->indexEntries, &singleDdtEntry); + TRACE("Added single-level DDT index entry at offset %" PRIu64, ctx->primaryDdtOffset); + } else TRACE("Failed to write single-level DDT table data to file"); } + + // Write the complete index at the end of the file + TRACE("Writing index at the end of the file"); + fseek(ctx->imageStream, 0, SEEK_END); + long indexPosition = ftell(ctx->imageStream); + + // Align index position to block boundary if needed + uint64_t alignmentMask = (1ULL << ctx->userDataDdtHeader.blockAlignmentShift) - 1; + if(indexPosition & alignmentMask) + { + uint64_t alignedPosition = (indexPosition + alignmentMask) & ~alignmentMask; + fseek(ctx->imageStream, alignedPosition, SEEK_SET); + indexPosition = alignedPosition; + TRACE("Aligned index position to %" PRIu64, alignedPosition); + } + + // Prepare index header + IndexHeader3 indexHeader; + indexHeader.identifier = IndexBlock3; + indexHeader.entries = utarray_len(ctx->indexEntries); + indexHeader.previous = 0; // No previous index for now + + TRACE("Writing index with %" PRIu64 " entries at position %ld", indexHeader.entries, indexPosition); + + // Calculate CRC64 of index entries + crc64_ctx *indexCrc64Context = aaruf_crc64_init(); + if(indexCrc64Context != NULL && indexHeader.entries > 0) + { + size_t indexDataSize = indexHeader.entries * sizeof(IndexEntry); + aaruf_crc64_update(indexCrc64Context, (uint8_t *)utarray_front(ctx->indexEntries), indexDataSize); + aaruf_crc64_final(indexCrc64Context, &indexHeader.crc64); + TRACE("Calculated index CRC64: 0x%16lX", indexHeader.crc64); + } + else { indexHeader.crc64 = 0; } + + // Write index header + if(fwrite(&indexHeader, sizeof(IndexHeader3), 1, ctx->imageStream) == 1) + { + TRACE("Successfully wrote index header"); + + // Write index entries + if(indexHeader.entries > 0) + { + size_t entriesWritten = 0; + IndexEntry *entry = NULL; + + for(entry = (IndexEntry *)utarray_front(ctx->indexEntries); entry != NULL; + entry = (IndexEntry *)utarray_next(ctx->indexEntries, entry)) + { + if(fwrite(entry, sizeof(IndexEntry), 1, ctx->imageStream) == 1) + { + entriesWritten++; + TRACE("Wrote index entry: blockType=0x%08X dataType=%u offset=%" PRIu64, entry->blockType, + entry->dataType, entry->offset); + } + else + { + TRACE("Failed to write index entry %zu", entriesWritten); + break; + } + } + + if(entriesWritten == indexHeader.entries) + { + TRACE("Successfully wrote all %zu index entries", entriesWritten); + + // Update header with index offset and rewrite it + ctx->header.indexOffset = indexPosition; + TRACE("Updating header with index offset: %" PRIu64, ctx->header.indexOffset); + + // Seek back to beginning and rewrite header + fseek(ctx->imageStream, 0, SEEK_SET); + if(fwrite(&ctx->header, sizeof(AaruHeaderV2), 1, ctx->imageStream) == 1) + { + TRACE("Successfully updated header with index offset"); + } + else + { + TRACE("Failed to update header with index offset"); + return AARUF_ERROR_CANNOT_WRITE_HEADER; + } + } + else + { + TRACE("Failed to write all index entries (wrote %zu of %" PRIu64 ")", entriesWritten, + indexHeader.entries); + return AARUF_ERROR_CANNOT_WRITE_HEADER; + } + } + } + else + { + TRACE("Failed to write index header"); + return AARUF_ERROR_CANNOT_WRITE_HEADER; + } } TRACE("Freeing memory pointers"); @@ -348,6 +507,13 @@ int aaruf_close(void *context) ctx->imageStream = NULL; } + // Free index entries array + if(ctx->indexEntries != NULL) + { + utarray_free(ctx->indexEntries); + ctx->indexEntries = NULL; + } + free(ctx->sectorPrefix); ctx->sectorPrefix = NULL; free(ctx->sectorPrefixCorrected); @@ -431,4 +597,4 @@ int aaruf_close(void *context) TRACE("Exiting aaruf_close() = 0"); return 0; -} \ No newline at end of file +} diff --git a/src/create.c b/src/create.c index a9fdf50..39eb3f9 100644 --- a/src/create.c +++ b/src/create.c @@ -165,6 +165,22 @@ void *aaruf_create(const char *filepath, uint32_t mediaType, uint32_t sectorSize if(ctx->userDataDdtHeader.blocks % (1 << ctx->userDataDdtHeader.tableShift) != 0) ctx->userDataDdtHeader.entries++; + // Initialize index entries array + TRACE("Initializing index entries array"); + UT_icd index_entry_icd = {sizeof(IndexEntry), NULL, NULL, NULL}; + utarray_new(ctx->indexEntries, &index_entry_icd); + + if(ctx->indexEntries == NULL) + { + FATAL("Not enough memory to create index entries array"); + free(ctx->readableSectorTags); + free(ctx); + errno = AARUF_ERROR_NOT_ENOUGH_MEMORY; + + TRACE("Exiting aaruf_create() = NULL"); + return NULL; + } + // Is writing ctx->isWriting = true; diff --git a/src/ddt/ddt_v2.c b/src/ddt/ddt_v2.c index a16b450..7b48d29 100644 --- a/src/ddt/ddt_v2.c +++ b/src/ddt/ddt_v2.c @@ -671,7 +671,7 @@ int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress if(buffer == NULL) { - FATAL(stderr, "Cannot allocate memory for DDT, stopping..."); + FATAL("Cannot allocate memory for DDT, stopping..."); TRACE("Exiting decode_ddt_multi_level_v2() = AARUF_ERROR_CANNOT_READ_BLOCK"); return AARUF_ERROR_CANNOT_READ_BLOCK; } @@ -977,6 +977,37 @@ void set_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool return; } + // Update index: remove old entry and add new one for the evicted secondary DDT + TRACE("Updating index for evicted secondary DDT"); + + // Remove old index entry for the cached DDT + if(ctx->cachedDdtOffset != 0) + { + TRACE("Removing old index entry for DDT at offset %" PRIu64, ctx->cachedDdtOffset); + IndexEntry *entry = NULL; + + // Find and remove the old index entry + for(unsigned int i = 0; i < utarray_len(ctx->indexEntries); i++) + { + entry = (IndexEntry *)utarray_eltptr(ctx->indexEntries, i); + if(entry && entry->offset == ctx->cachedDdtOffset && entry->blockType == DeDuplicationTable2) + { + TRACE("Found old DDT index entry at position %u, removing", i); + utarray_erase(ctx->indexEntries, i, 1); + break; + } + } + } + + // Add new index entry for the newly written secondary DDT + IndexEntry newDdtEntry; + newDdtEntry.blockType = DeDuplicationTable2; + newDdtEntry.dataType = UserData; + newDdtEntry.offset = endOfFile; + + utarray_push_back(ctx->indexEntries, &newDdtEntry); + TRACE("Added new DDT index entry at offset %" PRIu64, endOfFile); + // Step 4: Update the primary level table entry and flush it back to file uint64_t newSecondaryTableBlockOffset = endOfFile >> ctx->userDataDdtHeader.blockAlignmentShift; diff --git a/src/write.c b/src/write.c index 4d7cb44..a286228 100644 --- a/src/write.c +++ b/src/write.c @@ -153,7 +153,15 @@ int32_t aaruf_close_current_block(aaruformatContext *ctx) ctx->currentBlockHeader.cmpLength = ctx->currentBlockHeader.length; } - // TODO: Add to index + // Add to index + TRACE("Adding block to index"); + IndexEntry indexEntry; + indexEntry.blockType = DataBlock; + indexEntry.dataType = UserData; + indexEntry.offset = ctx->nextBlockPosition; + + utarray_push_back(ctx->indexEntries, &indexEntry); + TRACE("Block added to index at offset %" PRIu64, indexEntry.offset); // Write block header to file