aaru-dps/libaaruformat
1
0
Fork 0
mirror of https://github.com/aaru-dps/libaaruformat.git synced 2025-12-16 19:24:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement processing of DDT v2.

Browse Source
This commit is contained in:
Natalia Portillo
2025-08-04 16:31:29 +01:00
parent 6bf22a890c
commit 0e7dafc510
6 changed files with 448 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -116,7 +116,8 @@ add_library(aaruformat SHARED include/aaruformat/consts.h include/aaruformat/enu
src/blocks/optical.c
src/blocks/dump.c
src/blocks/checksum.c
src/index/index_v3.c)
src/index/index_v3.c
src/ddt/ddt_v2.c)
include_directories(include include/aaruformat)

58
include/aaruformat/context.h
View File

@@ -44,8 +44,8 @@ typedef struct Crc64Context
typedef struct CdEccContext
{
bool initedEdc;
uint8_t *eccBTable;
uint8_t *eccFTable;
uint8_t * eccBTable;
uint8_t * eccFTable;
uint32_t *edcTable;
} CdEccContext;
@@ -63,7 +63,7 @@ typedef struct Checksums
typedef struct mediaTagEntry
{
uint8_t *data;
uint8_t * data;
int32_t type;
uint32_t length;
UT_hash_handle hh;
@@ -74,52 +74,54 @@ typedef struct aaruformatContext
uint64_t magic;
uint8_t libraryMajorVersion;
uint8_t libraryMinorVersion;
FILE *imageStream;
FILE * imageStream;
AaruHeaderV2 header;
uint8_t *sectorPrefix;
uint8_t *sectorPrefixCorrected;
uint8_t *sectorSuffix;
uint8_t *sectorSuffixCorrected;
uint8_t *sectorSubchannel;
uint8_t *mode2Subheaders;
uint8_t * sectorPrefix;
uint8_t * sectorPrefixCorrected;
uint8_t * sectorSuffix;
uint8_t * sectorSuffixCorrected;
uint8_t * sectorSubchannel;
uint8_t * mode2Subheaders;
uint8_t shift;
bool inMemoryDdt;
uint64_t *userDataDdt;
uint64_t * userDataDdt;
size_t mappedMemoryDdtSize;
uint32_t *sectorPrefixDdt;
uint32_t *sectorSuffixDdt;
uint32_t * sectorPrefixDdt;
uint32_t * sectorSuffixDdt;
GeometryBlockHeader geometryBlock;
MetadataBlockHeader metadataBlockHeader;
uint8_t *metadataBlock;
uint8_t * metadataBlock;
TracksHeader tracksHeader;
TrackEntry *trackEntries;
TrackEntry * trackEntries;
CicmMetadataBlock cicmBlockHeader;
uint8_t *cicmBlock;
uint8_t * cicmBlock;
DumpHardwareHeader dumpHardwareHeader;
struct DumpHardwareEntriesWithData *dumpHardwareEntriesWithData;
struct ImageInfo imageInfo;
CdEccContext *eccCdContext;
CdEccContext * eccCdContext;
uint8_t numberOfDataTracks;
TrackEntry *dataTracks;
bool *readableSectorTags;
TrackEntry * dataTracks;
bool * readableSectorTags;
struct CacheHeader blockHeaderCache;
struct CacheHeader blockCache;
struct Checksums checksums;
struct mediaTagEntry *mediaTags;
struct mediaTagEntry * mediaTags;
DdtHeader2 userDataDdtHeader;
int ddtVersion;
} aaruformatContext;
typedef struct DumpHardwareEntriesWithData
{
DumpHardwareEntry entry;
struct DumpExtent *extents;
uint8_t *manufacturer;
uint8_t *model;
uint8_t *revision;
uint8_t *firmware;
uint8_t *serial;
uint8_t *softwareName;
uint8_t *softwareVersion;
uint8_t *softwareOperatingSystem;
uint8_t * manufacturer;
uint8_t * model;
uint8_t * revision;
uint8_t * firmware;
uint8_t * serial;
uint8_t * softwareName;
uint8_t * softwareVersion;
uint8_t * softwareOperatingSystem;
} DumpHardwareEntriesWithData;
#pragma pack(push, 1)

1
include/internal.h
View File

@@ -29,6 +29,7 @@ UT_array *process_index_v3(aaruformatContext *ctx);
int32_t verify_index_v3(aaruformatContext *ctx);
int32_t process_data_block(aaruformatContext *ctx, IndexEntry *entry);
int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt);
int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt);
void process_metadata_block(aaruformatContext *ctx, const IndexEntry *entry);
void process_geometry_block(aaruformatContext *ctx, const IndexEntry *entry);
void process_tracks_block(aaruformatContext *ctx, const IndexEntry *entry);

29
src/ddt/ddt_v1.c
View File

@@ -32,12 +32,11 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
int pos = 0;
size_t readBytes = 0;
DdtHeader ddtHeader;
uint8_t *cmpData = NULL;
uint8_t * cmpData = NULL;
uint32_t *cdDdt = NULL;
uint8_t lzmaProperties[LZMA_PROPERTIES_LENGTH];
size_t lzmaSize = 0;
int errorNo = 0;
BlockHeader blockHeader;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
@@ -74,6 +73,7 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
{
ctx->imageInfo.Sectors = ddtHeader.entries;
ctx->shift = ddtHeader.shift;
ctx->ddtVersion = 1;
// Check for DDT compression
switch(ddtHeader.compression)
@@ -160,13 +160,12 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
ctx->inMemoryDdt = false;
break;
#else // TODO: Implement
fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented...\n");
*foundUserDataDdt = false;
fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented...\n"); *foundUserDataDdt = false;
break;
#endif
default:
fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
blockHeader.compression);
ddtHeader.compression);
*foundUserDataDdt = false;
break;
}
@@ -236,12 +235,9 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(entry->dataType == CdSectorPrefixCorrected)
ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected)
ctx->sectorSuffixDdt = cdDdt;
else
free(cdDdt);
if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
else free(cdDdt);
break;
@@ -264,17 +260,14 @@ int32_t process_ddt_v1(aaruformatContext *ctx, IndexEntry *entry, bool *foundUse
break;
}
if(entry->dataType == CdSectorPrefixCorrected)
ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected)
ctx->sectorSuffixDdt = cdDdt;
else
free(cdDdt);
if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
else free(cdDdt);
break;
default:
fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
blockHeader.compression);
ddtHeader.compression);
break;
}
}

366
src/ddt/ddt_v2.c Normal file
View File

@@ -0,0 +1,366 @@
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2025 Natalia Portillo.
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include <inttypes.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef __linux__
#include <sys/mman.h>
#endif
#include "aaruformat.h"
int32_t process_ddt_v2(aaruformatContext *ctx, IndexEntry *entry, bool *foundUserDataDdt)
{
int pos = 0;
size_t readBytes = 0;
DdtHeader2 ddtHeader;
uint8_t * cmpData = NULL;
uint32_t * cdDdt = NULL;
uint8_t lzmaProperties[LZMA_PROPERTIES_LENGTH];
size_t lzmaSize = 0;
int errorNo = 0;
crc64_ctx *crc64_context = NULL;
uint64_t crc64 = 0;
// Check if the context and image stream are valid
if(ctx == NULL || ctx->imageStream == NULL)
{
fprintf(stderr, "Invalid context or image stream.\n");
return AARUF_ERROR_NOT_AARUFORMAT;
}
// Seek to block
pos = fseek(ctx->imageStream, entry->offset, SEEK_SET);
if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
{
fprintf(stderr, "libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry...\n", entry->offset);
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
// Even if those two checks shall have been done before
readBytes = fread(&ddtHeader, 1, sizeof(DdtHeader2), ctx->imageStream);
if(readBytes != sizeof(DdtHeader2))
{
fprintf(stderr, "libaaruformat: Could not read block header at %" PRIu64 "\n", entry->offset);
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
*foundUserDataDdt = false;
ctx->imageInfo.ImageSize += ddtHeader.cmpLength;
if(entry->dataType == UserData)
{
// User area sectors is blocks stored in DDT minus the negative and overflow displacement blocks
ctx->imageInfo.Sectors = ddtHeader.blocks - ddtHeader.negative - ddtHeader.overflow;
// We need the header later for the shift calculations
ctx->userDataDdtHeader = ddtHeader;
ctx->ddtVersion = 2;
// Check for DDT compression
switch(ddtHeader.compression)
{
case Lzma:
lzmaSize = ddtHeader.cmpLength - LZMA_PROPERTIES_LENGTH;
cmpData = (uint8_t *)malloc(lzmaSize);
if(cmpData == NULL)
{
fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
break;
}
ctx->userDataDdt = (uint64_t *)malloc(ddtHeader.length);
if(ctx->userDataDdt == NULL)
{
fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
free(cmpData);
break;
}
readBytes = fread(lzmaProperties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
if(readBytes != LZMA_PROPERTIES_LENGTH)
{
fprintf(stderr, "Could not read LZMA properties, continuing...\n");
free(cmpData);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
break;
}
readBytes = fread(cmpData, 1, lzmaSize, ctx->imageStream);
if(readBytes != lzmaSize)
{
fprintf(stderr, "Could not read compressed block, continuing...\n");
free(cmpData);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
break;
}
readBytes = ddtHeader.length;
errorNo = aaruf_lzma_decode_buffer((uint8_t *)ctx->userDataDdt, &readBytes, cmpData, &lzmaSize,
lzmaProperties, LZMA_PROPERTIES_LENGTH);
if(errorNo != 0)
{
fprintf(stderr, "Got error %d from LZMA, stopping...\n", errorNo);
free(cmpData);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(readBytes != ddtHeader.length)
{
fprintf(stderr, "Error decompressing block, should be {0} bytes but got {1} bytes., stopping...\n");
free(cmpData);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
free(cmpData);
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
fprintf(stderr, "Could not initialize CRC64.\n");
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, (uint8_t *)ctx->userDataDdt, readBytes);
aaruf_crc64_final(crc64_context, &crc64);
if(crc64 != ddtHeader.crc64)
{
fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
ctx->inMemoryDdt = true;
*foundUserDataDdt = true;
break;
case None:
ctx->userDataDdt = (uint64_t *)malloc(ddtHeader.length);
if(ctx->userDataDdt == NULL)
{
fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
free(cmpData);
break;
}
readBytes = fread(ctx->userDataDdt, 1, ddtHeader.entries * sizeof(uint32_t), ctx->imageStream);
if(readBytes != ddtHeader.entries * sizeof(uint32_t))
{
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
fprintf(stderr, "libaaruformat: Could not read deduplication table, continuing...\n");
break;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
fprintf(stderr, "Could not initialize CRC64.\n");
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, (uint8_t *)ctx->userDataDdt, readBytes);
aaruf_crc64_final(crc64_context, &crc64);
if(crc64 != ddtHeader.crc64)
{
fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
ctx->inMemoryDdt = true;
*foundUserDataDdt = true;
break;
default:
fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
ddtHeader.compression);
*foundUserDataDdt = false;
break;
}
}
else if(entry->dataType == CdSectorPrefixCorrected || entry->dataType == CdSectorSuffixCorrected)
{
switch(ddtHeader.compression)
{
case Lzma:
lzmaSize = ddtHeader.cmpLength - LZMA_PROPERTIES_LENGTH;
cmpData = (uint8_t *)malloc(lzmaSize);
if(cmpData == NULL)
{
fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
break;
}
cdDdt = (uint32_t *)malloc(ddtHeader.length);
if(cdDdt == NULL)
{
fprintf(stderr, "Cannot allocate memory for DDT, continuing...\n");
free(cmpData);
break;
}
readBytes = fread(lzmaProperties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
if(readBytes != LZMA_PROPERTIES_LENGTH)
{
fprintf(stderr, "Could not read LZMA properties, continuing...\n");
free(cmpData);
free(cdDdt);
ctx->userDataDdt = NULL;
break;
}
readBytes = fread(cmpData, 1, lzmaSize, ctx->imageStream);
if(readBytes != lzmaSize)
{
fprintf(stderr, "Could not read compressed block, continuing...\n");
free(cmpData);
free(cdDdt);
ctx->userDataDdt = NULL;
break;
}
readBytes = ddtHeader.length;
errorNo = aaruf_lzma_decode_buffer((uint8_t *)cdDdt, &readBytes, cmpData, &lzmaSize, lzmaProperties,
LZMA_PROPERTIES_LENGTH);
if(errorNo != 0)
{
fprintf(stderr, "Got error %d from LZMA, stopping...\n", errorNo);
free(cmpData);
free(cdDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
if(readBytes != ddtHeader.length)
{
fprintf(stderr, "Error decompressing block, should be {0} bytes but got {1} bytes., stopping...\n");
free(cmpData);
free(cdDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
fprintf(stderr, "Could not initialize CRC64.\n");
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, (uint8_t *)cdDdt, readBytes);
aaruf_crc64_final(crc64_context, &crc64);
if(crc64 != ddtHeader.crc64)
{
fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
else free(cdDdt);
break;
case None:
cdDdt = (uint32_t *)malloc(ddtHeader.entries * sizeof(uint32_t));
if(cdDdt == NULL)
{
fprintf(stderr, "libaaruformat: Cannot allocate memory for deduplication table.\n");
break;
}
readBytes = fread(cdDdt, 1, ddtHeader.entries * sizeof(uint32_t), ctx->imageStream);
if(readBytes != ddtHeader.entries * sizeof(uint32_t))
{
free(cdDdt);
fprintf(stderr, "libaaruformat: Could not read deduplication table, continuing...\n");
break;
}
crc64_context = aaruf_crc64_init();
if(crc64_context == NULL)
{
fprintf(stderr, "Could not initialize CRC64.\n");
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_CANNOT_READ_BLOCK;
}
aaruf_crc64_update(crc64_context, (uint8_t *)cdDdt, readBytes);
aaruf_crc64_final(crc64_context, &crc64);
if(crc64 != ddtHeader.crc64)
{
fprintf(stderr, "Expected DDT CRC 0x%16lX but got 0x%16lX.\n", ddtHeader.crc64, crc64);
free(ctx->userDataDdt);
ctx->userDataDdt = NULL;
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
if(entry->dataType == CdSectorPrefixCorrected) ctx->sectorPrefixDdt = cdDdt;
else if(entry->dataType == CdSectorSuffixCorrected) ctx->sectorSuffixDdt = cdDdt;
else free(cdDdt);
break;
default:
fprintf(stderr, "libaaruformat: Found unknown compression type %d, continuing...\n",
ddtHeader.compression);
break;
}
}
return AARUF_STATUS_OK;
}

35
src/open.c
View File

@@ -35,7 +35,7 @@ void *aaruf_open(const char *filepath)
long pos = 0;
int i = 0;
uint32_t signature = 0;
UT_array *index_entries = NULL;
UT_array * index_entries = NULL;
ctx = (aaruformatContext *)malloc(sizeof(aaruformatContext));
memset(ctx, 0, sizeof(aaruformatContext));
@@ -151,8 +151,8 @@ void *aaruf_open(const char *filepath)
readBytes = fread(&signature, 1, sizeof(uint32_t), ctx->imageStream);
if(readBytes != sizeof(uint32_t) ||
(signature != IndexBlock && signature != IndexBlock2 && signature != IndexBlock3))
if(readBytes != sizeof(uint32_t) || (signature != IndexBlock && signature != IndexBlock2 && signature !=
IndexBlock3))
{
free(ctx);
errno = AARUF_ERROR_CANNOT_READ_INDEX;
@@ -160,12 +160,9 @@ void *aaruf_open(const char *filepath)
return NULL;
}
if(signature == IndexBlock)
index_entries = process_index_v1(ctx);
else if(signature == IndexBlock2)
index_entries = process_index_v2(ctx);
else if(signature == IndexBlock3)
index_entries = process_index_v3(ctx);
if(signature == IndexBlock) index_entries = process_index_v1(ctx);
else if(signature == IndexBlock2) index_entries = process_index_v2(ctx);
else if(signature == IndexBlock3) index_entries = process_index_v3(ctx);
if(index_entries == NULL)
{
@@ -196,8 +193,8 @@ void *aaruf_open(const char *filepath)
if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
{
fprintf(stderr,
"libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry %d, continuing...\n",
fprintf(
stderr, "libaaruformat: Could not seek to %" PRIu64 " as indicated by index entry %d, continuing...\n",
entry->offset, i);
continue;
@@ -231,6 +228,19 @@ void *aaruf_open(const char *filepath)
return NULL;
}
break;
case DeDuplicationTable2:
errorNo = process_ddt_v2(ctx, entry, &foundUserDataDdt);
if(errorNo != AARUF_STATUS_OK)
{
utarray_free(index_entries);
free(ctx);
errno = errorNo;
return NULL;
}
break;
case GeometryBlock:
process_geometry_block(ctx, entry);
@@ -258,7 +268,8 @@ void *aaruf_open(const char *filepath)
break;
default:
fprintf(stderr,
fprintf(
stderr,
"libaaruformat: Unhandled block type %4.4s with data type %d is indexed to be at %" PRIu64 "\n",
(char *)&entry->blockType, entry->dataType, entry->offset);
break;