Add primary DDT offset and implement DDT entry setting functions

include/aaruformat/context.h

@@ -98,15 +98,15 @@ typedef struct aaruformatContext
     uint8_t                            *cicmBlock;
     DumpHardwareHeader                  dumpHardwareHeader;
     struct DumpHardwareEntriesWithData *dumpHardwareEntriesWithData;
-    struct ImageInfo                    imageInfo;
+    ImageInfo                           imageInfo;
     CdEccContext                       *eccCdContext;
     uint8_t                             numberOfDataTracks;
     TrackEntry                         *dataTracks;
     bool                               *readableSectorTags;
     struct CacheHeader                  blockHeaderCache;
     struct CacheHeader                  blockCache;
-    struct Checksums                    checksums;
-    struct mediaTagEntry               *mediaTags;
+    Checksums                           checksums;
+    mediaTagEntry                      *mediaTags;
     DdtHeader2                          userDataDdtHeader;
     int                                 ddtVersion;
     uint16_t                           *userDataDdtMini;
@@ -114,6 +114,7 @@ typedef struct aaruformatContext
     uint16_t                           *sectorPrefixDdtMini;
     uint16_t                           *sectorSuffixDdtMini;
     uint64_t                            cachedDdtOffset;
+    uint64_t                            primaryDdtOffset;
     uint16_t                           *cachedSecondaryDdtSmall;
     uint32_t                           *cachedSecondaryDdtBig;
     bool                                isWriting;

include/internal.h

@@ -45,6 +45,10 @@ int32_t   decode_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddr
                                      uint64_t *blockOffset, uint8_t *sectorStatus);
 int32_t   decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, uint64_t *offset,
                                     uint64_t *blockOffset, uint8_t *sectorStatus);
+void      set_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset,
+                                  uint64_t blockOffset, uint8_t sectorStatus);
+void      set_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset,
+                                 uint64_t blockOffset, uint8_t sectorStatus);
 aaru_options parse_options(const char *options);
 uint64_t     get_filetime_uint64();
 int32_t      aaruf_close_current_block(aaruformatContext *ctx);

src/ddt/ddt_v2.c

@@ -82,6 +82,8 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
         // We need the header later for the shift calculations
         ctx->userDataDdtHeader = ddtHeader;
         ctx->ddtVersion        = 2;
+        // Store the primary DDT table's file offset for secondary table references
+        ctx->primaryDdtOffset  = entry->offset;
 
         // Check for DDT compression
         switch(ddtHeader.compression)
@@ -358,7 +360,7 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
 
                 if(crc64_context == NULL)
                 {
-                    FATAL(stderr, "Could not initialize CRC64.");
+                    FATAL("Could not initialize CRC64.");
                     free(buffer);
                     TRACE("Exiting process_ddt_v2() = AARUF_ERROR_CANNOT_READ_BLOCK");
                     return AARUF_ERROR_CANNOT_READ_BLOCK;
@@ -371,6 +373,7 @@ int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
                 {
                     FATAL("Expected DDT CRC 0x%16lX but got 0x%16lX.", ddtHeader.crc64, crc64);
                     free(buffer);
+                    TRACE("Exiting process_ddt_v2() = AARUF_ERROR_INVALID_BLOCK_CRC");
                     return AARUF_ERROR_INVALID_BLOCK_CRC;
                 }
 
@@ -544,7 +547,7 @@ int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress
     }
 
     // Position in file of the child DDT table
-    secondaryDdtOffset *= (1 << ctx->userDataDdtHeader.blockAlignmentShift);
+    secondaryDdtOffset *= 1 << ctx->userDataDdtHeader.blockAlignmentShift;
 
     // Is the one we have cached the same as the one we need to read?
     if(ctx->cachedDdtOffset != secondaryDdtOffset)
@@ -753,4 +756,364 @@ int32_t decode_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress
     TRACE("Exiting decode_ddt_multi_level_v2(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx, sectorAddress,
           *offset, *blockOffset, *sectorStatus);
     return AARUF_STATUS_OK;
+}
+
+void set_ddt_entry_v2(aaruformatContext *ctx, uint64_t sectorAddress, uint64_t offset, uint64_t blockOffset,
+                      uint8_t sectorStatus)
+{
+    TRACE("Entering set_ddt_entry_v2(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sectorAddress, offset, blockOffset,
+          sectorStatus);
+
+    // Check if the context and image stream are valid
+    if(ctx == NULL || ctx->imageStream == NULL)
+    {
+        FATAL("Invalid context or image stream.");
+        return;
+    }
+
+    if(ctx->userDataDdtHeader.tableShift > 0)
+        set_ddt_multi_level_v2(ctx, sectorAddress, false, offset, blockOffset, sectorStatus);
+    else
+        set_ddt_single_level_v2(ctx, sectorAddress, false, offset, blockOffset, sectorStatus);
+}
+
+void set_ddt_single_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset,
+                             uint64_t blockOffset, uint8_t sectorStatus)
+{
+    TRACE("Entering set_ddt_single_level_v2(%p, %" PRIu64 ", %llu, %llu, %d)", ctx, sectorAddress, offset, blockOffset,
+          sectorStatus);
+
+    // Check if the context and image stream are valid
+    if(ctx == NULL || ctx->imageStream == NULL)
+    {
+        FATAL("Invalid context or image stream.");
+        return;
+    }
+
+    // Should not really be here
+    if(ctx->userDataDdtHeader.tableShift != 0)
+    {
+        FATAL("DDT table shift is not zero, but we are in single-level DDT setting.");
+        return;
+    }
+
+    // Calculate positive or negative sector
+    if(negative)
+        sectorAddress -= ctx->userDataDdtHeader.negative;
+    else
+        sectorAddress += ctx->userDataDdtHeader.negative;
+
+    uint64_t ddtEntry = 0;
+
+    uint64_t blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift;
+    ddtEntry = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex << ctx->userDataDdtHeader.dataShift;
+
+    if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+    {
+        ddtEntry |= (uint64_t)sectorStatus << 12;
+        ctx->cachedSecondaryDdtSmall[sectorAddress] = (uint16_t)ddtEntry;
+    }
+    else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType)
+    {
+        ddtEntry |= (uint64_t)sectorStatus << 28;
+        ctx->cachedSecondaryDdtBig[sectorAddress] = (uint32_t)ddtEntry;
+    }
+}
+
+void set_ddt_multi_level_v2(aaruformatContext *ctx, uint64_t sectorAddress, bool negative, uint64_t offset,
+                            uint64_t blockOffset, uint8_t sectorStatus)
+{
+    TRACE("Entering set_ddt_multi_level_v2(%p, %" PRIu64 ", %d, %" PRIu64 ", %" PRIu64 ", %d)", ctx, sectorAddress,
+          negative, offset, blockOffset, sectorStatus);
+
+    uint64_t   itemsPerDdtEntry   = 0;
+    uint64_t   ddtPosition        = 0;
+    uint64_t   secondaryDdtOffset = 0;
+    uint64_t   ddtEntry           = 0;
+    uint64_t   blockIndex         = 0;
+    uint8_t   *buffer             = NULL;
+    crc64_ctx *crc64_context      = NULL;
+    uint64_t   crc64              = 0;
+    DdtHeader2 ddtHeader;
+    size_t     writtenBytes   = 0;
+    long       currentPos     = 0;
+    long       endOfFile      = 0;
+    bool       createNewTable = false;
+
+    // Check if the context and image stream are valid
+    if(ctx == NULL || ctx->imageStream == NULL)
+    {
+        FATAL("Invalid context or image stream.");
+        return;
+    }
+
+    // Should not really be here
+    if(ctx->userDataDdtHeader.tableShift == 0)
+    {
+        FATAL("DDT table shift is zero, but we are in multi-level DDT setting.");
+        return;
+    }
+
+    // Calculate positive or negative sector
+    if(negative)
+        sectorAddress -= ctx->userDataDdtHeader.negative;
+    else
+        sectorAddress += ctx->userDataDdtHeader.negative;
+
+    // Step 1: Calculate the corresponding secondary level table
+    itemsPerDdtEntry = 1 << ctx->userDataDdtHeader.tableShift;
+    ddtPosition      = sectorAddress / itemsPerDdtEntry;
+
+    if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+        secondaryDdtOffset = ctx->userDataDdtMini[ddtPosition];
+    else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType)
+        secondaryDdtOffset = ctx->userDataDdtBig[ddtPosition];
+    else
+    {
+        FATAL("Unknown DDT size type %d.", ctx->userDataDdtHeader.sizeType);
+        return;
+    }
+
+    // Position in file of the child DDT table
+    secondaryDdtOffset *= 1 << ctx->userDataDdtHeader.blockAlignmentShift;
+
+    // Step 2: Check if it corresponds to the currently in-memory cached secondary level table
+    if(ctx->cachedDdtOffset == secondaryDdtOffset && secondaryDdtOffset != 0)
+    {
+        // Update the corresponding DDT entry directly in the cached table
+        blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift;
+        ddtEntry   = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex
+                                                                                 << ctx->userDataDdtHeader.dataShift;
+
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+        {
+            ddtEntry |= (uint64_t)sectorStatus << 12;
+            ctx->cachedSecondaryDdtSmall[sectorAddress % itemsPerDdtEntry] = (uint16_t)ddtEntry;
+        }
+        else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType)
+        {
+            ddtEntry |= (uint64_t)sectorStatus << 28;
+            ctx->cachedSecondaryDdtBig[sectorAddress % itemsPerDdtEntry] = (uint32_t)ddtEntry;
+        }
+
+        TRACE("Updated cached secondary DDT entry at position %" PRIu64, sectorAddress % itemsPerDdtEntry);
+        return;
+    }
+
+    // Step 3: Write the currently in-memory cached secondary level table to the end of the file
+    if(ctx->cachedDdtOffset != 0)
+    {
+        // Get current position and seek to end of file
+        currentPos = ftell(ctx->imageStream);
+        fseek(ctx->imageStream, 0, SEEK_END);
+        endOfFile = ftell(ctx->imageStream);
+
+        // Prepare DDT header for the cached table
+        memset(&ddtHeader, 0, sizeof(DdtHeader2));
+        ddtHeader.identifier          = DeDuplicationTable2;
+        ddtHeader.type                = UserData;
+        ddtHeader.compression         = None;  // Use no compression for simplicity
+        ddtHeader.levels              = ctx->userDataDdtHeader.levels;
+        ddtHeader.tableLevel          = ctx->userDataDdtHeader.tableLevel + 1;
+        ddtHeader.previousLevelOffset = ctx->primaryDdtOffset;  // Set to primary DDT table location
+        ddtHeader.negative            = ctx->userDataDdtHeader.negative;
+        ddtHeader.blocks              = itemsPerDdtEntry;
+        ddtHeader.overflow            = ctx->userDataDdtHeader.overflow;
+        ddtHeader.start               = ddtPosition * itemsPerDdtEntry;  // First block this DDT table references
+        ddtHeader.blockAlignmentShift = ctx->userDataDdtHeader.blockAlignmentShift;
+        ddtHeader.dataShift           = ctx->userDataDdtHeader.dataShift;
+        ddtHeader.tableShift          = 0;  // Secondary tables are single level
+        ddtHeader.sizeType            = ctx->userDataDdtHeader.sizeType;
+        ddtHeader.entries             = itemsPerDdtEntry;
+
+        // Calculate data size
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            ddtHeader.length = itemsPerDdtEntry * sizeof(uint16_t);
+        else
+            ddtHeader.length = itemsPerDdtEntry * sizeof(uint32_t);
+
+        ddtHeader.cmpLength = ddtHeader.length;
+
+        // Calculate CRC64 of the data
+        crc64_context = aaruf_crc64_init();
+        if(crc64_context == NULL)
+        {
+            FATAL("Could not initialize CRC64.");
+            return;
+        }
+
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cachedSecondaryDdtSmall, ddtHeader.length);
+        else
+            aaruf_crc64_update(crc64_context, (uint8_t *)ctx->cachedSecondaryDdtBig, ddtHeader.length);
+
+        aaruf_crc64_final(crc64_context, &crc64);
+        ddtHeader.crc64    = crc64;
+        ddtHeader.cmpCrc64 = crc64;
+
+        // Write header
+        writtenBytes = fwrite(&ddtHeader, sizeof(DdtHeader2), 1, ctx->imageStream);
+        if(writtenBytes != 1)
+        {
+            FATAL("Could not write DDT header to file.");
+            return;
+        }
+
+        // Write data
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            writtenBytes = fwrite(ctx->cachedSecondaryDdtSmall, ddtHeader.length, 1, ctx->imageStream);
+        else
+            writtenBytes = fwrite(ctx->cachedSecondaryDdtBig, ddtHeader.length, 1, ctx->imageStream);
+
+        if(writtenBytes != 1)
+        {
+            FATAL("Could not write DDT data to file.");
+            return;
+        }
+
+        // Step 4: Update the primary level table entry and flush it back to file
+        uint64_t newSecondaryTableBlockOffset = endOfFile >> ctx->userDataDdtHeader.blockAlignmentShift;
+
+        // Find which entry in the primary table corresponds to the cached secondary table
+        uint64_t cachedDdtPosition = ctx->cachedDdtOffset >> ctx->userDataDdtHeader.blockAlignmentShift;
+
+        // Update the primary table entry to point to the new location of the secondary table
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            ctx->userDataDdtMini[cachedDdtPosition] = (uint16_t)newSecondaryTableBlockOffset;
+        else
+            ctx->userDataDdtBig[cachedDdtPosition] = (uint32_t)newSecondaryTableBlockOffset;
+
+        // Write the updated primary table back to its original position in the file
+        long savedPos = ftell(ctx->imageStream);
+        fseek(ctx->imageStream, ctx->primaryDdtOffset + sizeof(DdtHeader2), SEEK_SET);
+
+        size_t primaryTableSize = ctx->userDataDdtHeader.sizeType == SmallDdtSizeType
+                                      ? ctx->userDataDdtHeader.entries * sizeof(uint16_t)
+                                      : ctx->userDataDdtHeader.entries * sizeof(uint32_t);
+
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            writtenBytes = fwrite(ctx->userDataDdtMini, primaryTableSize, 1, ctx->imageStream);
+        else
+            writtenBytes = fwrite(ctx->userDataDdtBig, primaryTableSize, 1, ctx->imageStream);
+
+        if(writtenBytes != 1)
+        {
+            FATAL("Could not flush primary DDT table to file.");
+            return;
+        }
+
+        fseek(ctx->imageStream, savedPos, SEEK_SET);
+
+        // Free the cached table
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType && ctx->cachedSecondaryDdtSmall)
+        {
+            free(ctx->cachedSecondaryDdtSmall);
+            ctx->cachedSecondaryDdtSmall = NULL;
+        }
+        else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType && ctx->cachedSecondaryDdtBig)
+        {
+            free(ctx->cachedSecondaryDdtBig);
+            ctx->cachedSecondaryDdtBig = NULL;
+        }
+
+        // Restore file position
+        fseek(ctx->imageStream, currentPos, SEEK_SET);
+    }
+
+    // Step 5: Check if the specified block already has an existing secondary level table
+    createNewTable = secondaryDdtOffset == 0;
+
+    if(!createNewTable)
+    {
+        // Load existing table
+        fseek(ctx->imageStream, secondaryDdtOffset, SEEK_SET);
+        size_t readBytes = fread(&ddtHeader, 1, sizeof(DdtHeader2), ctx->imageStream);
+
+        if(readBytes != sizeof(DdtHeader2) || ddtHeader.identifier != DeDuplicationTable2 || ddtHeader.type != UserData)
+        {
+            FATAL("Invalid secondary DDT header at %" PRIu64, secondaryDdtOffset);
+            return;
+        }
+
+        // Read the table data (assuming no compression for now)
+        buffer = malloc(ddtHeader.length);
+        if(buffer == NULL)
+        {
+            FATAL("Cannot allocate memory for secondary DDT.");
+            return;
+        }
+
+        readBytes = fread(buffer, 1, ddtHeader.length, ctx->imageStream);
+        if(readBytes != ddtHeader.length)
+        {
+            FATAL("Could not read secondary DDT data.");
+            free(buffer);
+            return;
+        }
+
+        // Verify CRC
+        crc64_context = aaruf_crc64_init();
+        if(crc64_context == NULL)
+        {
+            FATAL("Could not initialize CRC64.");
+            free(buffer);
+            return;
+        }
+
+        aaruf_crc64_update(crc64_context, buffer, readBytes);
+        aaruf_crc64_final(crc64_context, &crc64);
+
+        if(crc64 != ddtHeader.crc64)
+        {
+            FATAL("Secondary DDT CRC mismatch. Expected 0x%16lX but got 0x%16lX.", ddtHeader.crc64, crc64);
+            free(buffer);
+            return;
+        }
+
+        // Cache the loaded table
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            ctx->cachedSecondaryDdtSmall = (uint16_t *)buffer;
+        else
+            ctx->cachedSecondaryDdtBig = (uint32_t *)buffer;
+
+        ctx->cachedDdtOffset = secondaryDdtOffset;
+    }
+    else
+    {
+        // Create a new empty table
+        size_t tableSize = ctx->userDataDdtHeader.sizeType == SmallDdtSizeType ? itemsPerDdtEntry * sizeof(uint16_t)
+                                                                               : itemsPerDdtEntry * sizeof(uint32_t);
+
+        buffer = calloc(1, tableSize);
+        if(buffer == NULL)
+        {
+            FATAL("Cannot allocate memory for new secondary DDT.");
+            return;
+        }
+
+        if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+            ctx->cachedSecondaryDdtSmall = (uint16_t *)buffer;
+        else
+            ctx->cachedSecondaryDdtBig = (uint32_t *)buffer;
+
+        ctx->cachedDdtOffset = 0;  // Will be set when written to file
+    }
+
+    // Step 6: Update the corresponding DDT entry
+    blockIndex = blockOffset >> ctx->userDataDdtHeader.blockAlignmentShift;
+    ddtEntry = offset & (1ULL << ctx->userDataDdtHeader.dataShift) - 1 | blockIndex << ctx->userDataDdtHeader.dataShift;
+
+    if(ctx->userDataDdtHeader.sizeType == SmallDdtSizeType)
+    {
+        ddtEntry |= (uint64_t)sectorStatus << 12;
+        ctx->cachedSecondaryDdtSmall[sectorAddress % itemsPerDdtEntry] = (uint16_t)ddtEntry;
+    }
+    else if(ctx->userDataDdtHeader.sizeType == BigDdtSizeType)
+    {
+        ddtEntry |= (uint64_t)sectorStatus << 28;
+        ctx->cachedSecondaryDdtBig[sectorAddress % itemsPerDdtEntry] = (uint32_t)ddtEntry;
+    }
+
+    TRACE("Updated secondary DDT entry at position %" PRIu64, sectorAddress % itemsPerDdtEntry);
+    TRACE("Exiting set_ddt_multi_level_v2()");
 }