Implement processing of DDT v2.

CMakeLists.txt

@@ -116,7 +116,8 @@ add_library(aaruformat SHARED include/aaruformat/consts.h include/aaruformat/enu
             src/blocks/optical.c
             src/blocks/dump.c
             src/blocks/checksum.c
-            src/index/index_v3.c)
+            src/index/index_v3.c
+            src/ddt/ddt_v2.c)
 
 include_directories(include include/aaruformat)
 

include/aaruformat/context.h

@@ -106,6 +106,8 @@ typedef struct aaruformatContext
     struct CacheHeader                  blockCache;
     struct Checksums                    checksums;
     struct mediaTagEntry *              mediaTags;
+    DdtHeader2                          userDataDdtHeader;
+    int                                 ddtVersion;
 } aaruformatContext;
 
 typedef struct DumpHardwareEntriesWithData

include/internal.h

@@ -29,6 +29,7 @@ UT_array *process_index_v3(aaruformatContext *ctx);
 int32_t   verify_index_v3(aaruformatContext *ctx);
 int32_t   process_data_block(aaruformatContext *ctx, IndexEntry *entry);
 int32_t   process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt);
+int32_t   process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt);
 void      process_metadata_block(aaruformatContext *ctx, const IndexEntry *entry);
 void      process_geometry_block(aaruformatContext *ctx, const IndexEntry *entry);
 void      process_tracks_block(aaruformatContext *ctx, const IndexEntry *entry);

src/ddt/ddt_v1.c

@@ -37,7 +37,6 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
     uint8_t   lzmaProperties[LZMA_PROPERTIES_LENGTH];
     size_t    lzmaSize = 0;
     int       errorNo  = 0;
-    BlockHeader blockHeader;
 
     // Check if the context and image stream are valid
     if(ctx == NULL || ctx->imageStream == NULL)
@@ -74,6 +73,7 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
     {
         ctx->imageInfo.Sectors = ddtHeader.entries;
         ctx->shift             = ddtHeader.shift;
+        ctx->ddtVersion        = 1;
 
         // Check for DDT compression
         switch(ddtHeader.compression)
@@ -160,13 +160,12 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
                 ctx->inMemoryDdt = false;
                 break;
 #else  // TODO: Implement
-                fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented...\n");
-                *foundUserDataDdt = false;
+                fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented...\n"); *foundUserDataDdt = false;
                 break;
 #endif
             default:
                 fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
-                        blockHeader.compression);
+                        ddtHeader.compression);
                 *foundUserDataDdt = false;
                 break;
         }
@@ -236,12 +235,9 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
                     return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
                 }
 
-                if(entry->dataType == CdSectorPrefixCorrected)
-                    ctx->sectorPrefixDdt = cdDdt;
-                else if(entry->dataType == CdSectorSuffixCorrected)
-                    ctx->sectorSuffixDdt = cdDdt;
-                else
-                    free(cdDdt);
+                if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
+                else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
+                else free(cdDdt);
 
                 break;
 
@@ -264,17 +260,14 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
                     break;
                 }
 
-                if(entry->dataType == CdSectorPrefixCorrected)
-                    ctx->sectorPrefixDdt = cdDdt;
-                else if(entry->dataType == CdSectorSuffixCorrected)
-                    ctx->sectorSuffixDdt = cdDdt;
-                else
-                    free(cdDdt);
+                if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
+                else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
+                else free(cdDdt);
 
                 break;
             default:
                 fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
-                        blockHeader.compression);
+                        ddtHeader.compression);
                 break;
         }
     }

src/ddt/ddt_v2.c

@@ -0,0 +1,366 @@
+/*
+ * This file is part of the Aaru Data Preservation Suite.
+ * Copyright (c) 2019-2025 Natalia Portillo.
+ *
+ * This library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/mman.h>
+#endif
+
+#include "aaruformat.h"
+
+int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt)
+{
+    int        pos       = 0;
+    size_t     readBytes = 0;
+    DdtHeader2 ddtHeader;
+    uint8_t *  cmpData = NULL;
+    uint32_t * cdDdt   = NULL;
+    uint8_t    lzmaProperties[LZMA_PROPERTIES_LENGTH];
+    size_t     lzmaSize      = 0;
+    int        errorNo       = 0;
+    crc64_ctx *crc64_context = NULL;
+    uint64_t   crc64         = 0;
+
+    // Check if the context and image stream are valid
+    if(ctx == NULL || ctx->imageStream == NULL)
+    {
+        fprintf(stderr, "Invalid context or image stream.\n");
+        return AARUF_ERROR_NOT_AARUFORMAT;
+    }
+
+    // Seek to block
+    pos = fseek(ctx->imageStream, entry->offset, SEEK_SET);
+    if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
+    {
+        fprintf(stderr, "libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry...\n", entry->offset);
+
+        return AARUF_ERROR_CANNOT_READ_BLOCK;
+    }
+
+    // Even if those two checks shall have been done before
+
+    readBytes = fread(&ddtHeader, 1, sizeof(DdtHeader2), ctx->imageStream);
+
+    if(readBytes != sizeof(DdtHeader2))
+    {
+        fprintf(stderr, "libaaruformat: Could not read block header at %" PRIu64 "\n", entry->offset);
+
+        return AARUF_ERROR_CANNOT_READ_BLOCK;
+    }
+
+    *foundUserDataDdt = false;
+
+    ctx->imageInfo.ImageSize += ddtHeader.cmpLength;
+
+    if(entry->dataType == UserData)
+    {
+        // User area sectors is blocks stored in DDT minus the negative and overflow displacement blocks
+        ctx->imageInfo.Sectors = ddtHeader.blocks - ddtHeader.negative - ddtHeader.overflow;
+        // We need the header later for the shift calculations
+        ctx->userDataDdtHeader = ddtHeader;
+        ctx->ddtVersion        = 2;
+
+        // Check for DDT compression
+        switch(ddtHeader.compression)
+        {
+            case Lzma:
+                lzmaSize = ddtHeader.cmpLength - LZMA_PROPERTIES_LENGTH;
+
+                cmpData = (uint8_t *)malloc(lzmaSize);
+                if(cmpData == NULL)
+                {
+                    fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
+                    break;
+                }
+
+                ctx->userDataDdt = (uint64_t *)malloc(ddtHeader.length);
+                if(ctx->userDataDdt == NULL)
+                {
+                    fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
+                    free(cmpData);
+                    break;
+                }
+
+                readBytes = fread(lzmaProperties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
+                if(readBytes != LZMA_PROPERTIES_LENGTH)
+                {
+                    fprintf(stderr, "Could not read LZMA properties, continuing...\n");
+                    free(cmpData);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    break;
+                }
+
+                readBytes = fread(cmpData, 1, lzmaSize, ctx->imageStream);
+                if(readBytes != lzmaSize)
+                {
+                    fprintf(stderr, "Could not read compressed block, continuing...\n");
+                    free(cmpData);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    break;
+                }
+
+                readBytes = ddtHeader.length;
+                errorNo   = aaruf_lzma_decode_buffer((uint8_t *)ctx->userDataDdt, &readBytes, cmpData, &lzmaSize,
+                                                     lzmaProperties, LZMA_PROPERTIES_LENGTH);
+
+                if(errorNo != 0)
+                {
+                    fprintf(stderr, "Got error %d from LZMA, stopping...\n", errorNo);
+                    free(cmpData);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
+                }
+
+                if(readBytes != ddtHeader.length)
+                {
+                    fprintf(stderr, "Error decompressing block, should be {0} bytes but got {1} bytes., stopping...\n");
+                    free(cmpData);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
+                }
+
+                free(cmpData);
+
+                crc64_context = aaruf_crc64_init();
+
+                if(crc64_context == NULL)
+                {
+                    fprintf(stderr, "Could not initialize CRC64.\n");
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_READ_BLOCK;
+                }
+
+                aaruf_crc64_update(crc64_context, (uint8_t *)ctx->userDataDdt, readBytes);
+                aaruf_crc64_final(crc64_context, &crc64);
+
+                if(crc64 != ddtHeader.crc64)
+                {
+                    fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_INVALID_BLOCK_CRC;
+                }
+
+                ctx->inMemoryDdt  = true;
+                *foundUserDataDdt = true;
+
+                break;
+            case None:
+                ctx->userDataDdt = (uint64_t *)malloc(ddtHeader.length);
+                if(ctx->userDataDdt == NULL)
+                {
+                    fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
+                    free(cmpData);
+                    break;
+                }
+
+                readBytes = fread(ctx->userDataDdt, 1, ddtHeader.entries * sizeof(uint32_t), ctx->imageStream);
+
+                if(readBytes != ddtHeader.entries * sizeof(uint32_t))
+                {
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    fprintf(stderr, "libaaruformat: Could not read deduplication table, continuing...\n");
+                    break;
+                }
+
+                crc64_context = aaruf_crc64_init();
+
+                if(crc64_context == NULL)
+                {
+                    fprintf(stderr, "Could not initialize CRC64.\n");
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_READ_BLOCK;
+                }
+
+                aaruf_crc64_update(crc64_context, (uint8_t *)ctx->userDataDdt, readBytes);
+                aaruf_crc64_final(crc64_context, &crc64);
+
+                if(crc64 != ddtHeader.crc64)
+                {
+                    fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_INVALID_BLOCK_CRC;
+                }
+
+                ctx->inMemoryDdt  = true;
+                *foundUserDataDdt = true;
+
+                break;
+            default:
+                fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
+                        ddtHeader.compression);
+                *foundUserDataDdt = false;
+                break;
+        }
+    }
+    else if(entry->dataType == CdSectorPrefixCorrected || entry->dataType == CdSectorSuffixCorrected)
+    {
+        switch(ddtHeader.compression)
+        {
+            case Lzma:
+                lzmaSize = ddtHeader.cmpLength - LZMA_PROPERTIES_LENGTH;
+
+                cmpData = (uint8_t *)malloc(lzmaSize);
+                if(cmpData == NULL)
+                {
+                    fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
+                    break;
+                }
+
+                cdDdt = (uint32_t *)malloc(ddtHeader.length);
+                if(cdDdt == NULL)
+                {
+                    fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
+                    free(cmpData);
+                    break;
+                }
+
+                readBytes = fread(lzmaProperties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
+                if(readBytes != LZMA_PROPERTIES_LENGTH)
+                {
+                    fprintf(stderr, "Could not read LZMA properties, continuing...\n");
+                    free(cmpData);
+                    free(cdDdt);
+                    ctx->userDataDdt = NULL;
+                    break;
+                }
+
+                readBytes = fread(cmpData, 1, lzmaSize, ctx->imageStream);
+                if(readBytes != lzmaSize)
+                {
+                    fprintf(stderr, "Could not read compressed block, continuing...\n");
+                    free(cmpData);
+                    free(cdDdt);
+                    ctx->userDataDdt = NULL;
+                    break;
+                }
+
+                readBytes = ddtHeader.length;
+                errorNo   = aaruf_lzma_decode_buffer((uint8_t *)cdDdt, &readBytes, cmpData, &lzmaSize, lzmaProperties,
+                                                     LZMA_PROPERTIES_LENGTH);
+
+                if(errorNo != 0)
+                {
+                    fprintf(stderr, "Got error %d from LZMA, stopping...\n", errorNo);
+                    free(cmpData);
+                    free(cdDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
+                }
+
+                if(readBytes != ddtHeader.length)
+                {
+                    fprintf(stderr, "Error decompressing block, should be {0} bytes but got {1} bytes., stopping...\n");
+                    free(cmpData);
+                    free(cdDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
+                }
+
+                crc64_context = aaruf_crc64_init();
+
+                if(crc64_context == NULL)
+                {
+                    fprintf(stderr, "Could not initialize CRC64.\n");
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_READ_BLOCK;
+                }
+
+                aaruf_crc64_update(crc64_context, (uint8_t *)cdDdt, readBytes);
+                aaruf_crc64_final(crc64_context, &crc64);
+
+                if(crc64 != ddtHeader.crc64)
+                {
+                    fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_INVALID_BLOCK_CRC;
+                }
+
+                if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
+                else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
+                else free(cdDdt);
+
+                break;
+
+            case None:
+                cdDdt = (uint32_t *)malloc(ddtHeader.entries * sizeof(uint32_t));
+
+                if(cdDdt == NULL)
+                {
+                    fprintf(stderr, "libaaruformat: Cannot allocate memory for deduplication table.\n");
+                    break;
+                }
+
+                readBytes = fread(cdDdt, 1, ddtHeader.entries * sizeof(uint32_t), ctx->imageStream);
+
+                if(readBytes != ddtHeader.entries * sizeof(uint32_t))
+                {
+                    free(cdDdt);
+                    fprintf(stderr, "libaaruformat: Could not read deduplication table, continuing...\n");
+                    break;
+                }
+
+                crc64_context = aaruf_crc64_init();
+
+                if(crc64_context == NULL)
+                {
+                    fprintf(stderr, "Could not initialize CRC64.\n");
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_CANNOT_READ_BLOCK;
+                }
+
+                aaruf_crc64_update(crc64_context, (uint8_t *)cdDdt, readBytes);
+                aaruf_crc64_final(crc64_context, &crc64);
+
+                if(crc64 != ddtHeader.crc64)
+                {
+                    fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
+                    free(ctx->userDataDdt);
+                    ctx->userDataDdt = NULL;
+                    return AARUF_ERROR_INVALID_BLOCK_CRC;
+                }
+
+                if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
+                else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
+                else free(cdDdt);
+
+                break;
+            default:
+                fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
+                        ddtHeader.compression);
+                break;
+        }
+    }
+
+    return AARUF_STATUS_OK;
+}

src/open.c

@@ -151,8 +151,8 @@ void *aaruf_open(const char *filepath)
 
     readBytes = fread(&signature, 1, sizeof(uint32_t), ctx->imageStream);
 
-    if(readBytes != sizeof(uint32_t) ||
-       (signature != IndexBlock && signature != IndexBlock2 && signature != IndexBlock3))
+    if(readBytes != sizeof(uint32_t) || (signature != IndexBlock && signature != IndexBlock2 && signature !=
+                                         IndexBlock3))
     {
         free(ctx);
         errno = AARUF_ERROR_CANNOT_READ_INDEX;
@@ -160,12 +160,9 @@ void *aaruf_open(const char *filepath)
         return NULL;
     }
 
-    if(signature == IndexBlock)
-        index_entries = process_index_v1(ctx);
-    else if(signature == IndexBlock2)
-        index_entries = process_index_v2(ctx);
-    else if(signature == IndexBlock3)
-        index_entries = process_index_v3(ctx);
+    if(signature == IndexBlock) index_entries = process_index_v1(ctx);
+    else if(signature == IndexBlock2) index_entries = process_index_v2(ctx);
+    else if(signature == IndexBlock3) index_entries = process_index_v3(ctx);
 
     if(index_entries == NULL)
     {
@@ -196,8 +193,8 @@ void *aaruf_open(const char *filepath)
 
         if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
         {
-            fprintf(stderr,
-                    "libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry %d, continuing...\n",
+            fprintf(
+                stderr, "libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry %d, continuing...\n",
                 entry->offset, i);
 
             continue;
@@ -231,6 +228,19 @@ void *aaruf_open(const char *filepath)
                     return NULL;
                 }
 
+                break;
+            case DeDuplicationTable2:
+                errorNo = process_ddt_v2(ctx, entry, &foundUserDataDdt);
+
+                if(errorNo != AARUF_STATUS_OK)
+                {
+                    utarray_free(index_entries);
+                    free(ctx);
+                    errno = errorNo;
+
+                    return NULL;
+                }
+
                 break;
             case GeometryBlock:
                 process_geometry_block(ctx, entry);
@@ -258,7 +268,8 @@ void *aaruf_open(const char *filepath)
 
                 break;
             default:
-                fprintf(stderr,
+                fprintf(
+                    stderr,
                     "libaaruformat: Unhandled block type %4.4s with data type %d is indexed to be at %" PRIu64 "\n",
                     (char *)&entry->blockType, entry->dataType, entry->offset);
                 break;