libaaruformat 1.0 Aaru Data Preservation Suite - Format Library

Loading...
Searching...
No Matches
ddt_v1.c
Go to the documentation of this file.
1/*
2 * This file is part of the Aaru Data Preservation Suite.
3 * Copyright (c) 2019-2025 Natalia Portillo.
4 *
5 * This library is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU Lesser General Public License as
7 * published by the Free Software Foundation; either version 2.1 of the
8 * License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <inttypes.h>
20#include <stdint.h>
21#include <stdio.h>
22#include <stdlib.h>
23#include "errors.h"
24
25#ifdef __linux__
26#include <sys/mman.h>
27#endif
28
29#include "aaruformat.h"
30#include "log.h"
31
85int32_t process_ddt_v1(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
86{
87 TRACE("Entering process_ddt_v1(%p, %p, %d)", ctx, entry, *found_user_data_ddt);
88
89 int pos = 0;
90 size_t read_bytes = 0;
91 DdtHeader ddt_header;
92 uint8_t *cmp_data = NULL;
93 uint32_t *cd_ddt = NULL;
94 uint8_t lzma_properties[LZMA_PROPERTIES_LENGTH];
95 size_t lzma_size = 0;
96 int error_no = 0;
97
98 // Check if the context and image stream are valid
99 if(ctx == NULL || ctx->imageStream == NULL)
100 {
101 FATAL("Invalid context or image stream.");
102
103 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_NOT_AARUFORMAT");
104 return AARUF_ERROR_NOT_AARUFORMAT;
105 }
106
107 // Seek to block
108 TRACE("Seeking to DDT block at position %" PRIu64, entry->offset);
109 pos = fseek(ctx->imageStream, entry->offset, SEEK_SET);
110 if(pos < 0 || ftell(ctx->imageStream) != entry->offset)
111 {
112 FATAL("Could not seek to %" PRIu64 " as indicated by index entry...", entry->offset);
113
114 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_CANNOT_READ_BLOCK");
115 return AARUF_ERROR_CANNOT_READ_BLOCK;
116 }
117
118 // Even if those two checks shall have been done before
119 TRACE("Reading DDT block header at position %" PRIu64, entry->offset);
120 read_bytes = fread(&ddt_header, 1, sizeof(DdtHeader), ctx->imageStream);
121
122 if(read_bytes != sizeof(DdtHeader))
123 {
124 FATAL("Could not read block header at %" PRIu64 "", entry->offset);
125
126 TRACE("Exiting process_ddt_v1() = AARUF_ERROR_CANNOT_READ_BLOCK");
127 return AARUF_ERROR_CANNOT_READ_BLOCK;
128 }
129
130 *found_user_data_ddt = true;
131
132 ctx->image_info.ImageSize += ddt_header.cmpLength;
133
134 if(entry->dataType == UserData)
135 {
136 ctx->image_info.Sectors = ddt_header.entries;
137 ctx->shift = ddt_header.shift;
138 ctx->ddt_version = 1;
139
140 // Check for DDT compression
141 switch(ddt_header.compression)
142 {
143 // TODO: Check CRC
144 case Lzma:
145 if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
146 {
147 FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
148 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
149 }
150
151 lzma_size = ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH;
152
153 cmp_data = (uint8_t *)malloc(lzma_size);
154 if(cmp_data == NULL)
155 {
156 TRACE("Cannot allocate memory for DDT, continuing...");
157 break;
158 }
159
160 ctx->user_data_ddt = (uint64_t *)malloc(ddt_header.length);
161 if(ctx->user_data_ddt == NULL)
162 {
163 TRACE("Cannot allocate memory for DDT, continuing...");
164 free(cmp_data);
165 break;
166 }
167
168 read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
169 if(read_bytes != LZMA_PROPERTIES_LENGTH)
170 {
171 TRACE("Could not read LZMA properties, continuing...");
172 free(cmp_data);
173 free(ctx->user_data_ddt);
174 ctx->user_data_ddt = NULL;
175 break;
176 }
177
178 read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
179 if(read_bytes != lzma_size)
180 {
181 TRACE("Could not read compressed block, continuing...");
182 free(cmp_data);
183 free(ctx->user_data_ddt);
184 ctx->user_data_ddt = NULL;
185 break;
186 }
187
188 read_bytes = ddt_header.length;
189 TRACE("Decompressing block of size %zu bytes", ddt_header.length);
190 error_no = aaruf_lzma_decode_buffer((uint8_t *)ctx->user_data_ddt, &read_bytes, cmp_data, &lzma_size,
191 lzma_properties, LZMA_PROPERTIES_LENGTH);
192
193 if(error_no != 0)
194 {
195 FATAL("Got error %d from LZMA, stopping...", error_no);
196 free(cmp_data);
197 free(ctx->user_data_ddt);
198 ctx->user_data_ddt = NULL;
199 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
200 }
201
202 if(read_bytes != ddt_header.length)
203 {
204 FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
205 free(cmp_data);
206 free(ctx->user_data_ddt);
207 ctx->user_data_ddt = NULL;
208 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
209 }
210
211 free(cmp_data);
212 cmp_data = NULL;
213
214 ctx->in_memory_ddt = true;
215 *found_user_data_ddt = true;
216
217 break;
218 // TODO: Check CRC
219 case None:
220#ifdef __linux__
221 TRACE("Memory mapping deduplication table at position %" PRIu64, entry->offset + sizeof(ddt_header));
222 ctx->mapped_memory_ddt_size = sizeof(uint64_t) * ddt_header.entries;
223 ctx->user_data_ddt = mmap(NULL, ctx->mapped_memory_ddt_size, PROT_READ, MAP_SHARED,
224 fileno(ctx->imageStream), entry->offset + sizeof(ddt_header));
225
226 if(ctx->user_data_ddt == MAP_FAILED)
227 {
228 *found_user_data_ddt = false;
229 FATAL("Could not read map deduplication table.");
230 break;
231 }
232
233 ctx->in_memory_ddt = false;
234 break;
235#else // TODO: Implement
236 TRACE("Uncompressed DDT not yet implemented...");
237 *found_user_data_ddt = false;
238 break;
239#endif
240 default:
241 TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
242 *found_user_data_ddt = false;
243 break;
244 }
245 }
246 else if(entry->dataType == CdSectorPrefixCorrected || entry->dataType == CdSectorSuffixCorrected)
247 {
248 switch(ddt_header.compression)
249 {
250 // TODO: Check CRC
251 case Lzma:
252 if(ddt_header.cmpLength <= LZMA_PROPERTIES_LENGTH)
253 {
254 FATAL("Compressed DDT payload too small (%" PRIu64 ") for LZMA properties.", ddt_header.cmpLength);
255 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
256 }
257
258 lzma_size = ddt_header.cmpLength - LZMA_PROPERTIES_LENGTH;
259
260 cmp_data = (uint8_t *)malloc(lzma_size);
261 if(cmp_data == NULL)
262 {
263 TRACE("Cannot allocate memory for DDT, continuing...");
264 break;
265 }
266
267 cd_ddt = (uint32_t *)malloc(ddt_header.length);
268 if(cd_ddt == NULL)
269 {
270 TRACE("Cannot allocate memory for DDT, continuing...");
271 free(cmp_data);
272 break;
273 }
274
275 read_bytes = fread(lzma_properties, 1, LZMA_PROPERTIES_LENGTH, ctx->imageStream);
276 if(read_bytes != LZMA_PROPERTIES_LENGTH)
277 {
278 TRACE("Could not read LZMA properties, continuing...");
279 free(cmp_data);
280 free(cd_ddt);
281 break;
282 }
283
284 read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
285 if(read_bytes != lzma_size)
286 {
287 TRACE("Could not read compressed block, continuing...");
288 free(cmp_data);
289 free(cd_ddt);
290 break;
291 }
292
293 read_bytes = ddt_header.length;
294 TRACE("Decompressing block of size %zu bytes", ddt_header.length);
295 error_no = aaruf_lzma_decode_buffer((uint8_t *)cd_ddt, &read_bytes, cmp_data, &lzma_size,
296 lzma_properties, LZMA_PROPERTIES_LENGTH);
297
298 if(error_no != 0)
299 {
300 FATAL("Got error %d from LZMA, stopping...", error_no);
301 free(cmp_data);
302 free(cd_ddt);
303 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
304 }
305
306 if(read_bytes != ddt_header.length)
307 {
308 FATAL("Error decompressing block, should be {0} bytes but got {1} bytes., stopping...");
309 free(cmp_data);
310 free(cd_ddt);
311 return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
312 }
313
314 free(cmp_data);
315 cmp_data = NULL;
316
317 if(entry->dataType == CdSectorPrefixCorrected)
318 ctx->sector_prefix_ddt = cd_ddt;
319 else if(entry->dataType == CdSectorSuffixCorrected)
320 ctx->sector_suffix_ddt = cd_ddt;
321 else
322 free(cd_ddt);
323
324 break;
325
326 // TODO: Check CRC
327 case None:
328 cd_ddt = (uint32_t *)malloc(ddt_header.entries * sizeof(uint32_t));
329
330 if(cd_ddt == NULL)
331 {
332 TRACE("Cannot allocate memory for deduplication table.");
333 break;
334 }
335
336 read_bytes = fread(cd_ddt, 1, ddt_header.entries * sizeof(uint32_t), ctx->imageStream);
337
338 if(read_bytes != ddt_header.entries * sizeof(uint32_t))
339 {
340 free(cd_ddt);
341 TRACE("Could not read deduplication table, continuing...");
342 break;
343 }
344
345 if(entry->dataType == CdSectorPrefixCorrected)
346 ctx->sector_prefix_ddt = cd_ddt;
347 else if(entry->dataType == CdSectorSuffixCorrected)
348 ctx->sector_suffix_ddt = cd_ddt;
349 else
350 free(cd_ddt);
351
352 break;
353 default:
354 TRACE("Found unknown compression type %d, continuing...", ddt_header.compression);
355 break;
356 }
357 }
358
359 TRACE("Exiting process_ddt_v1() = AARUF_STATUS_OK");
360 return AARUF_STATUS_OK;
361}
362
405int32_t decode_ddt_entry_v1(aaruformat_context *ctx, const uint64_t sector_address, uint64_t *offset,
406 uint64_t *block_offset, uint8_t *sector_status)
407{
408 TRACE("Entering decode_ddt_entry_v1(%p, %" PRIu64 ", %p, %p, %p)", ctx, sector_address, offset, block_offset,
409 sector_status);
410
411 // Check if the context and image stream are valid
412 if(ctx == NULL || ctx->imageStream == NULL)
413 {
414 FATAL("Invalid context or image stream.");
415 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_NOT_AARUFORMAT");
416 return AARUF_ERROR_NOT_AARUFORMAT;
417 }
418
419 if(ctx->user_data_ddt == NULL)
420 {
421 FATAL("User data DDT not loaded.");
422 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_NOT_AARUFORMAT");
423 return AARUF_ERROR_NOT_AARUFORMAT;
424 }
425
426 if(ctx->shift >= 64)
427 {
428 FATAL("Invalid DDT shift value %u", ctx->shift);
429 TRACE("Exiting decode_ddt_entry_v1() = AARUF_ERROR_INCORRECT_DATA_SIZE");
430 return AARUF_ERROR_INCORRECT_DATA_SIZE;
431 }
432
433 const uint64_t ddt_entry = ctx->user_data_ddt[sector_address];
434 const uint64_t offset_mask64 = (UINT64_C(1) << ctx->shift) - UINT64_C(1);
435 *offset = ddt_entry & offset_mask64;
436 *block_offset = ddt_entry >> ctx->shift;
437
438 // Partially written image... as we can't know the real sector size just assume it's common :/
439 if(ddt_entry == 0)
440 *sector_status = SectorStatusNotDumped;
441 else
442 *sector_status = SectorStatusDumped;
443
444 TRACE("Exiting decode_ddt_entry_v1(%p, %" PRIu64 ", %llu, %llu, %d) = AARUF_STATUS_OK", ctx, sector_address,
445 *offset, *block_offset, *sector_status);
446 return AARUF_STATUS_OK;
447}
aaruformat.h
LZMA_PROPERTIES_LENGTH
#define LZMA_PROPERTIES_LENGTH
Size in bytes of the fixed LZMA properties header (lc/lp/pb + dictionary size).
Definition consts.h:82
decode_ddt_entry_v1
int32_t decode_ddt_entry_v1(aaruformat_context *ctx, const uint64_t sector_address, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status)
Decodes a DDT v1 entry for a given sector address.
Definition ddt_v1.c:405
process_ddt_v1
int32_t process_ddt_v1(aaruformat_context *ctx, IndexEntry *entry, bool *found_user_data_ddt)
Processes a DDT v1 block from the image stream.
Definition ddt_v1.c:85
aaruf_lzma_decode_buffer
int32_t aaruf_lzma_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t *src_size, const uint8_t *props, size_t props_size)
Decodes an LZMA-compressed buffer.
Definition lzma.c:39
SectorStatusDumped
@ SectorStatusDumped
Sector(s) successfully dumped without error.
Definition enums.h:231
SectorStatusNotDumped
@ SectorStatusNotDumped
Sector(s) not yet acquired during image dumping.
Definition enums.h:230
CdSectorPrefixCorrected
@ CdSectorPrefixCorrected
Compact Disc sector prefix (sync, header) corrected-only stored.
Definition enums.h:121
CdSectorSuffixCorrected
@ CdSectorSuffixCorrected
Compact Disc sector suffix (EDC, ECC P, ECC Q) corrected-only stored.
Definition enums.h:122
UserData
@ UserData
User (main) data.
Definition enums.h:46
Lzma
@ Lzma
LZMA compression.
Definition enums.h:34
None
@ None
Not compressed.
Definition enums.h:33
errors.h
Public error and status code definitions for libaaruformat.
AARUF_STATUS_OK
#define AARUF_STATUS_OK
Sector present and read without uncorrectable errors.
Definition errors.h:75
AARUF_ERROR_CANNOT_READ_BLOCK
#define AARUF_ERROR_CANNOT_READ_BLOCK
Generic block read failure (seek/read error).
Definition errors.h:46
AARUF_ERROR_INCORRECT_DATA_SIZE
#define AARUF_ERROR_INCORRECT_DATA_SIZE
Data size does not match expected size.
Definition errors.h:65
AARUF_ERROR_NOT_AARUFORMAT
#define AARUF_ERROR_NOT_AARUFORMAT
Input file/stream failed magic or structural validation.
Definition errors.h:40
AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
#define AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK
Decompression routine failed or size mismatch.
Definition errors.h:56
log.h
FATAL
#define FATAL(fmt,...)
Definition log.h:40
TRACE
#define TRACE(fmt,...)
Definition log.h:25
DdtHeader
Header preceding a version 1 (flat) deduplication table body.
Definition ddt.h:66
DdtHeader::shift
uint8_t shift
Left shift applied to per-entry file offset component forming logicalEntryValue.
Definition ddt.h:70
DdtHeader::compression
uint16_t compression
Compression algorithm for the table body (CompressionType).
Definition ddt.h:69
DdtHeader::cmpLength
uint64_t cmpLength
Size in bytes of compressed entries payload.
Definition ddt.h:72
DdtHeader::length
uint64_t length
Size in bytes of uncompressed entries payload.
Definition ddt.h:73
DdtHeader::entries
uint64_t entries
Number of deduplication entries contained in (uncompressed) table.
Definition ddt.h:71
ImageInfo::ImageSize
uint64_t ImageSize
Size of the image payload in bytes (excludes headers/metadata)
Definition aaru.h:873
ImageInfo::Sectors
uint64_t Sectors
Total count of addressable logical sectors/blocks.
Definition aaru.h:874
IndexEntry
Single index entry describing a block's type, (optional) data classification, and file offset.
Definition index.h:109
IndexEntry::offset
uint64_t offset
Absolute byte offset in the image where the referenced block header begins.
Definition index.h:112
IndexEntry::dataType
uint16_t dataType
Data classification (value from DataType) or unused for untyped blocks.
Definition index.h:111
aaruformat_context
Master context representing an open or in‑creation Aaru image.
Definition context.h:172
aaruformat_context::user_data_ddt
uint64_t * user_data_ddt
Legacy flat DDT pointer (NULL when using v2 mini/big arrays).
Definition context.h:181
aaruformat_context::shift
uint8_t shift
Legacy overall shift (deprecated by data_shift/table_shift).
Definition context.h:195
aaruformat_context::sector_suffix_ddt
uint32_t * sector_suffix_ddt
Legacy CD sector suffix DDT.
Definition context.h:184
aaruformat_context::in_memory_ddt
bool in_memory_ddt
True if primary (and possibly secondary) DDT loaded.
Definition context.h:196
aaruformat_context::ddt_version
int ddt_version
DDT version in use (1=legacy, 2=v2 hierarchical).
Definition context.h:194
aaruformat_context::imageStream
FILE * imageStream
Underlying FILE* stream (binary mode).
Definition context.h:176
aaruformat_context::image_info
ImageInfo image_info
Exposed high-level image info summary.
Definition context.h:260
aaruformat_context::sector_prefix_ddt
uint32_t * sector_prefix_ddt
Legacy CD sector prefix DDT (deprecated by *2).
Definition context.h:183
aaruformat_context::mapped_memory_ddt_size
size_t mapped_memory_ddt_size
Length of mmapped DDT if userDataDdt is mmapped.
Definition context.h:193