libaaruformat 1.0
Aaru Data Preservation Suite - Format Library
Loading...
Searching...
No Matches
ddt_v1.c
Go to the documentation of this file.
1/*
2 * This file is part of the Aaru Data Preservation Suite.
3 * Copyright (c) 2019-2026 Natalia Portillo.
4 *
5 * This library is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU Lesser General Public License as
7 * published by the Free Software Foundation; either version 2.1 of the
8 * License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <inttypes.h>
20#include <stdint.h>
21#include <stdio.h>
22#include <stdlib.h>
23#include "errors.h"
24
25#ifdef __linux__
26#include <sys/mman.h>
27#endif
28
29#include "aaruformat.h"
30#include "internal.h"
31#include "log.h"
32
86int32_t process_ddt_v1(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
87{
88 TRACE("Entering process_ddt_v1(%p, %p, %d)", ctx, entry, *found_user_data_ddt);
89
90 int pos = 0;
91 size_t read_bytes = 0;
92 DdtHeader ddt_header;
93 uint8_t *cmp_data = NULL;
94 uint32_t *cd_ddt = NULL;
95 uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH];
96 size_t lzma_size = 0;
97 int error_no = 0;
98
99 // Check if the context and image stream are valid
100 if(ctx == NULL || ctx->imageStream == NULL)
101 {
102 FATAL("Invalid context or image stream.");
103
104 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_NOT_AARUFORMAT");
105 return AARUF_ERROR_NOT_AARUFORMAT;
106 }
107
108 // Seek to block
109 TRACE("Seeking to DDT block at position %" PRIu64, entry->offset);
110 pos = aaruf_fseek(ctx->imageStream, (aaru_off_t)entry->offset, SEEK_SET);
111 if(pos < 0 || aaruf_ftell(ctx->imageStream) != (aaru_off_t)entry->offset)
112 {
113 FATAL("Could not seek to %" PRIu64 " as indicated by index entry...", entry->offset);
114
115 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_CANNOT_READ_BLOCK");
116 return AARUF_ERROR_CANNOT_READ_BLOCK;
117 }
118
119 // Even if those two checks shall have been done before
120 TRACE("Reading DDT block header at position %" PRIu64, entry->offset);
121 read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader), ctx->imageStream);
122
123 if(read_bytes != sizeof(DdtHeader))
124 {
125 FATAL("Could not read block header at %" PRIu64 "", entry->offset);
126
127 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_CANNOT_READ_BLOCK");
128 return AARUF_ERROR_CANNOT_READ_BLOCK;
129 }
130
131 *found_user_data_ddt = true;
132
133 ctx->image_info.ImageSize += ddt_header.cmpLength;
134
135 if(entry->dataType == kDataTypeUserData)
136 {
137 ctx->image_info.Sectors = ddt_header.entries;
138 ctx->shift = ddt_header.shift;
139 ctx->ddt_version = 1;
140
141 // Check for DDT compression
142 switch(ddt_header.compression)
143 {
144 // TODO: Check CRC
145 case kCompressionLzma:
146 if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
147 {
148 FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
149 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
150 }
151
152 lzma_size = ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH;
153
154 cmp_data = (uint8_t *)malloc(lzma_size);
155 if(cmp_data == NULL)
156 {
157 TRACE("Cannot allocate memory for DDT, continuing...");
158 break;
159 }
160
161 ctx->user_data_ddt = (uint64_t *)malloc(ddt_header.length);
162 if(ctx->user_data_ddt == NULL)
163 {
164 TRACE("Cannot allocate memory for DDT, continuing...");
165 free(cmp_data);
166 break;
167 }
168
169 read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
170 if(read_bytes != LZMA_PROPERTIES_LENGTH)
171 {
172 TRACE("Could not read LZMA properties, continuing...");
173 free(cmp_data);
174 free(ctx->user_data_ddt);
175 ctx->user_data_ddt = NULL;
176 break;
177 }
178
179 read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
180 if(read_bytes != lzma_size)
181 {
182 TRACE("Could not read compressed block, continuing...");
183 free(cmp_data);
184 free(ctx->user_data_ddt);
185 ctx->user_data_ddt = NULL;
186 break;
187 }
188
189 read_bytes = ddt_header.length;
190 TRACE("Decompressing block of size %zu bytes", ddt_header.length);
191 error_no = aaruf_lzma_decode_buffer((uint8_t *)ctx->user_data_ddt, &read_bytes, cmp_data, &lzma_size,
192 lzma_properties, LZMA_PROPERTIES_LENGTH);
193
194 if(error_no != 0)
195 {
196 FATAL("Got error %d from LZMA, stopping...", error_no);
197 free(cmp_data);
198 free(ctx->user_data_ddt);
199 ctx->user_data_ddt = NULL;
200 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
201 }
202
203 if(read_bytes != ddt_header.length)
204 {
205 FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
206 free(cmp_data);
207 free(ctx->user_data_ddt);
208 ctx->user_data_ddt = NULL;
209 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
210 }
211
212 free(cmp_data);
213 cmp_data = NULL;
214
215 ctx->in_memory_ddt = true;
216 *found_user_data_ddt = true;
217
218 break;
219 case kCompressionZstd:
220 cmp_data = (uint8_t *)malloc(ddt_header.cmpLength);
221 if(cmp_data == NULL)
222 {
223 TRACE("Cannot allocate memory for DDT, continuing...");
224 break;
225 }
226
227 ctx->user_data_ddt = (uint64_t *)malloc(ddt_header.length);
228 if(ctx->user_data_ddt == NULL)
229 {
230 TRACE("Cannot allocate memory for DDT, continuing...");
231 free(cmp_data);
232 break;
233 }
234
235 read_bytes = fread(cmp_data, 1, ddt_header.cmpLength, ctx->imageStream);
236 if(read_bytes != ddt_header.cmpLength)
237 {
238 TRACE("Could not read compressed block, continuing...");
239 free(cmp_data);
240 free(ctx->user_data_ddt);
241 ctx->user_data_ddt = NULL;
242 break;
243 }
244
245 read_bytes = aaruf_zstd_decode_buffer((uint8_t *)ctx->user_data_ddt, ddt_header.length, cmp_data,
246 ddt_header.cmpLength);
247 if(read_bytes != ddt_header.length)
248 {
249 FATAL("Error decompressing zstd DDT block, expected %zu got %zu", ddt_header.length, read_bytes);
250 free(cmp_data);
251 free(ctx->user_data_ddt);
252 ctx->user_data_ddt = NULL;
253 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
254 }
255
256 free(cmp_data);
257 cmp_data = NULL;
258
259 ctx->in_memory_ddt = true;
260 *found_user_data_ddt = true;
261
262 break;
263 // TODO: Check CRC
264 case kCompressionNone:
265 ctx->user_data_ddt = (uint64_t *)malloc(ddt_header.length);
266 if(ctx->user_data_ddt == NULL)
267 {
268 TRACE("Cannot allocate memory for DDT, continuing...");
269 break;
270 }
271
272 read_bytes = fread(ctx->user_data_ddt, 1, ddt_header.entries * sizeof(uint64_t), ctx->imageStream);
273
274 if(read_bytes != ddt_header.entries * sizeof(uint64_t))
275 {
276 free(ctx->user_data_ddt);
277 TRACE("Could not read deduplication table, continuing...");
278 break;
279 }
280
281 ctx->in_memory_ddt = true;
282 *found_user_data_ddt = true;
283
284 break;
285 default:
286 TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
287 *found_user_data_ddt = false;
288 break;
289 }
290 }
291 else if(entry->dataType == kDataTypeCdSectorPrefixCorrected || entry->dataType == kDataTypeCdSectorSuffixCorrected)
292 {
293 switch(ddt_header.compression)
294 {
295 // TODO: Check CRC
296 case kCompressionLzma:
297 if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
298 {
299 FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
300 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
301 }
302
303 lzma_size = ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH;
304
305 cmp_data = (uint8_t *)malloc(lzma_size);
306 if(cmp_data == NULL)
307 {
308 TRACE("Cannot allocate memory for DDT, continuing...");
309 break;
310 }
311
312 cd_ddt = (uint32_t *)malloc(ddt_header.length);
313 if(cd_ddt == NULL)
314 {
315 TRACE("Cannot allocate memory for DDT, continuing...");
316 free(cmp_data);
317 break;
318 }
319
320 read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
321 if(read_bytes != LZMA_PROPERTIES_LENGTH)
322 {
323 TRACE("Could not read LZMA properties, continuing...");
324 free(cmp_data);
325 free(cd_ddt);
326 break;
327 }
328
329 read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
330 if(read_bytes != lzma_size)
331 {
332 TRACE("Could not read compressed block, continuing...");
333 free(cmp_data);
334 free(cd_ddt);
335 break;
336 }
337
338 read_bytes = ddt_header.length;
339 TRACE("Decompressing block of size %zu bytes", ddt_header.length);
340 error_no = aaruf_lzma_decode_buffer((uint8_t *)cd_ddt, &read_bytes, cmp_data, &lzma_size,
341 lzma_properties, LZMA_PROPERTIES_LENGTH);
342
343 if(error_no != 0)
344 {
345 FATAL("Got error %d from LZMA, stopping...", error_no);
346 free(cmp_data);
347 free(cd_ddt);
348 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
349 }
350
351 if(read_bytes != ddt_header.length)
352 {
353 FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
354 free(cmp_data);
355 free(cd_ddt);
356 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
357 }
358
359 free(cmp_data);
360 cmp_data = NULL;
361
362 if(entry->dataType == kDataTypeCdSectorPrefixCorrected)
363 ctx->sector_prefix_ddt = cd_ddt;
364 else if(entry->dataType == kDataTypeCdSectorSuffixCorrected)
365 ctx->sector_suffix_ddt = cd_ddt;
366 else
367 free(cd_ddt);
368
369 break;
370 case kCompressionZstd:
371 cmp_data = (uint8_t *)malloc(ddt_header.cmpLength);
372 if(cmp_data == NULL)
373 {
374 TRACE("Cannot allocate memory for DDT, continuing...");
375 break;
376 }
377
378 cd_ddt = (uint32_t *)malloc(ddt_header.length);
379 if(cd_ddt == NULL)
380 {
381 TRACE("Cannot allocate memory for DDT, continuing...");
382 free(cmp_data);
383 break;
384 }
385
386 read_bytes = fread(cmp_data, 1, ddt_header.cmpLength, ctx->imageStream);
387 if(read_bytes != ddt_header.cmpLength)
388 {
389 TRACE("Could not read compressed block, continuing...");
390 free(cmp_data);
391 free(cd_ddt);
392 break;
393 }
394
395 read_bytes = aaruf_zstd_decode_buffer((uint8_t *)cd_ddt, ddt_header.length, cmp_data,
396 ddt_header.cmpLength);
397 if(read_bytes != ddt_header.length)
398 {
399 FATAL("Error decompressing zstd DDT block, expected %zu got %zu", ddt_header.length, read_bytes);
400 free(cmp_data);
401 free(cd_ddt);
402 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
403 }
404
405 free(cmp_data);
406 cmp_data = NULL;
407
408 if(entry->dataType == kDataTypeCdSectorPrefixCorrected)
409 ctx->sector_prefix_ddt = cd_ddt;
410 else if(entry->dataType == kDataTypeCdSectorSuffixCorrected)
411 ctx->sector_suffix_ddt = cd_ddt;
412 else
413 free(cd_ddt);
414
415 break;
416
417 // TODO: Check CRC
418 case kCompressionNone:
419 cd_ddt = (uint32_t *)malloc(ddt_header.entries * sizeof(uint32_t));
420
421 if(cd_ddt == NULL)
422 {
423 TRACE("Cannot allocate memory for deduplication table.");
424 break;
425 }
426
427 read_bytes = fread(cd_ddt, 1, ddt_header.entries * sizeof(uint32_t), ctx->imageStream);
428
429 if(read_bytes != ddt_header.entries * sizeof(uint32_t))
430 {
431 free(cd_ddt);
432 TRACE("Could not read deduplication table, continuing...");
433 break;
434 }
435
436 if(entry->dataType == kDataTypeCdSectorPrefixCorrected)
437 ctx->sector_prefix_ddt = cd_ddt;
438 else if(entry->dataType == kDataTypeCdSectorSuffixCorrected)
439 ctx->sector_suffix_ddt = cd_ddt;
440 else
441 free(cd_ddt);
442
443 break;
444 default:
445 TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
446 break;
447 }
448 }
449
450 TRACE("Exiting process_ddt_v1() = AARUF_STATUS_OK");
451 return AARUF_STATUS_OK;
452}
453
496int32_t decode_ddt_entry_v1(aaruformat_context *ctx, const uint64_t sector_address, uint64_t *offset,
497 uint64_t *block_offset, uint8_t *sector_status)
498{
499 TRACE("Entering decode_ddt_entry_v1(%p, %" PRIu64 ", %p, %p, %p)", ctx, sector_address, offset, block_offset,
500 sector_status);
501
502 // Check if the context and image stream are valid
503 if(ctx == NULL || ctx->imageStream == NULL)
504 {
505 FATAL("Invalid context or image stream.");
506 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_NOT_AARUFORMAT");
507 return AARUF_ERROR_NOT_AARUFORMAT;
508 }
509
510 if(ctx->user_data_ddt == NULL)
511 {
512 FATAL("User data DDT not loaded.");
513 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_NOT_AARUFORMAT");
514 return AARUF_ERROR_NOT_AARUFORMAT;
515 }
516
517 if(ctx->shift >= 64)
518 {
519 FATAL("Invalid DDT shift value %u", ctx->shift);
520 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_INCORRECT_DATA_SIZE");
521 return AARUF_ERROR_INCORRECT_DATA_SIZE;
522 }
523
524 const uint64_t ddt_entry = ctx->user_data_ddt[sector_address];
525 const uint64_t offset_mask64 = (UINT64_C(1) << ctx->shift) - UINT64_C(1);
526 *offset = ddt_entry & offset_mask64;
527 *block_offset = ddt_entry >> ctx->shift;
528
529 // Partially written image... as we can't know the real sector size just assume it's common :/
530 if(ddt_entry == 0)
531 *sector_status = SectorStatusNotDumped;
532 else
533 *sector_status = SectorStatusDumped;
534
535 TRACE("Exiting decode_ddt_entry_v1(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx, sector_address,
536 *offset, *block_offset, *sector_status);
537 return AARUF_STATUS_OK;
538}
LZMA_PROPERTIES_LENGTH
#define LZMA_PROPERTIES_LENGTH
Size in bytes of the fixed LZMA properties header (lc/lp/pb + dictionary size).
Definition consts.h:82
decode_ddt_entry_v1
int32_t decode_ddt_entry_v1(aaruformat_context *ctx, const uint64_t sector_address, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status)
Decodes a DDT v1 entry for a given sector address.
Definition ddt_v1.c:496
process_ddt_v1
int32_t process_ddt_v1(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
Processes a DDT v1 block from the image stream.
Definition ddt_v1.c:86
aaruf_lzma_decode_buffer
int32_t aaruf_lzma_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t *src_size, const uint8_t *props, size_t props_size)
Decodes an LZMA-compressed buffer.
Definition lzma.c:39
aaruf_zstd_decode_buffer
size_t aaruf_zstd_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, size_t src_size)
Decodes a Zstandard-compressed buffer.
Definition zstd.c:34
SectorStatusDumped
@ SectorStatusDumped
Sector(s) successfully dumped without error.
Definition enums.h:259
SectorStatusNotDumped
@ SectorStatusNotDumped
Sector(s) not yet acquired during image dumping.
Definition enums.h:258
kDataTypeCdSectorSuffixCorrected
@ kDataTypeCdSectorSuffixCorrected
Compact Disc sector suffix (EDC, ECC P, ECC Q) corrected-only stored.
Definition enums.h:124
kDataTypeUserData
@ kDataTypeUserData
User (main) data.
Definition enums.h:48
kDataTypeCdSectorPrefixCorrected
@ kDataTypeCdSectorPrefixCorrected
Compact Disc sector prefix (sync, header) corrected-only stored.
Definition enums.h:123
kCompressionLzma
@ kCompressionLzma
LZMA compression.
Definition enums.h:34
kCompressionNone
@ kCompressionNone
Not compressed.
Definition enums.h:33
kCompressionZstd
@ kCompressionZstd
Zstandard compression.
Definition enums.h:37
errors.h
Public error and status code definitions for libaaruformat.
AARUF_STATUS_OK
#define AARUF_STATUS_OK
Sector present and read without uncorrectable errors.
Definition errors.h:81
AARUF_ERROR_CANNOT_READ_BLOCK
#define AARUF_ERROR_CANNOT_READ_BLOCK
Generic block read failure (seek/read error).
Definition errors.h:46
AARUF_ERROR_INCORRECT_DATA_SIZE
#define AARUF_ERROR_INCORRECT_DATA_SIZE
Data size does not match expected size.
Definition errors.h:65
AARUF_ERROR_NOT_AARUFORMAT
#define AARUF_ERROR_NOT_AARUFORMAT
Input file/stream failed magic or structural validation.
Definition errors.h:40
AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
#define AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
Decompression routine failed or size mismatch.
Definition errors.h:56
aaruf_fseek
static int aaruf_fseek(FILE *stream, aaru_off_t offset, int origin)
Definition internal.h:46
aaruf_ftell
static aaru_off_t aaruf_ftell(FILE *stream)
Definition internal.h:52
aaru_off_t
int64_t aaru_off_t
Definition internal.h:42
FATAL
#define FATAL(fmt,...)
Definition log.h:40
TRACE
#define TRACE(fmt,...)
Definition log.h:25
DdtHeader
Header preceding a version 1 (flat) deduplication table body.
Definition ddt.h:66
DdtHeader::shift
uint8_t shift
Left shift applied to per-entry file offset component forming logicalEntryValue.
Definition ddt.h:70
DdtHeader::compression
uint16_t compression
Compression algorithm for the table body (CompressionType).
Definition ddt.h:69
DdtHeader::cmpLength
uint64_t cmpLength
Size in bytes of compressed entries payload.
Definition ddt.h:72
DdtHeader::length
uint64_t length
Size in bytes of uncompressed entries payload.
Definition ddt.h:73
DdtHeader::entries
uint64_t entries
Number of deduplication entries contained in (uncompressed) table.
Definition ddt.h:71
ImageInfo::ImageSize
uint64_t ImageSize
Size of the image payload in bytes (excludes headers/metadata).
Definition aaru.h:937
ImageInfo::Sectors
uint64_t Sectors
Total count of addressable logical sectors/blocks.
Definition aaru.h:938
IndexEntry
Single index entry describing a block's type, (optional) data classification, and file offset.
Definition index.h:109
IndexEntry::offset
uint64_t offset
Absolute byte offset in the image where the referenced block header begins.
Definition index.h:112
IndexEntry::dataType
uint16_t dataType
Data classification (value from DataType) or unused for untyped blocks.
Definition index.h:111
aaruformat_context
Master context representing an open or in‑creation Aaru image.
Definition context.h:175
aaruformat_context::user_data_ddt
uint64_t * user_data_ddt
Legacy flat DDT pointer (NULL when using v2 mini/big arrays).
Definition context.h:184
aaruformat_context::shift
uint8_t shift
Legacy overall shift (deprecated by data_shift/table_shift).
Definition context.h:198
aaruformat_context::sector_suffix_ddt
uint32_t * sector_suffix_ddt
Legacy CD sector suffix DDT.
Definition context.h:187
aaruformat_context::in_memory_ddt
bool in_memory_ddt
True if primary (and possibly secondary) DDT loaded.
Definition context.h:199
aaruformat_context::ddt_version
int ddt_version
DDT version in use (1=legacy, 2=v2 hierarchical).
Definition context.h:197
aaruformat_context::imageStream
FILE * imageStream
Underlying FILE* stream (binary mode).
Definition context.h:179
aaruformat_context::image_info
ImageInfo image_info
Exposed high-level image info summary.
Definition context.h:263
aaruformat_context::sector_prefix_ddt
uint32_t * sector_prefix_ddt
Legacy CD sector prefix DDT (deprecated by *2).
Definition context.h:186