mirror of
https://github.com/aaru-dps/libaaruformat.git
synced 2026-07-08 18:06:18 +00:00
Use EC to recover damaged data blocks.
This commit is contained in:
@@ -389,6 +389,13 @@ typedef struct aaruformat_context
|
||||
uint32_t ec_total_data_blocks; ///< Total data blocks written (counter for round-robin assignment).
|
||||
UT_array *ec_data_stripes; ///< Completed data stripe descriptors (serialized to ECMB).
|
||||
bool ec_enabled; ///< True if erasure coding is active.
|
||||
|
||||
/* Erasure coding (read path) */
|
||||
void *ec_read_stripes; ///< Parsed EcReadStripe array for data group, NULL if no ECMB.
|
||||
uint32_t ec_read_stripe_count; ///< Number of data stripes parsed from ECMB.
|
||||
void *ec_block_lookup; ///< uthash: block file offset → stripe index + position.
|
||||
bool ec_recovery_available; ///< True if ECMB loaded and recovery is possible.
|
||||
bool ec_recovery_in_progress; ///< Recursion guard for recovery (prevents infinite loops).
|
||||
} aaruformat_context;
|
||||
|
||||
#ifndef AARUFORMAT_CONTEXT_DECLARED
|
||||
|
||||
@@ -22,31 +22,45 @@
|
||||
#include "aaruformat/context.h"
|
||||
#include "aaruformat/structs/data.h"
|
||||
|
||||
/**
|
||||
* @brief Accumulate parity for a data block just written to disk.
|
||||
*
|
||||
* Called from aaruf_close_current_block() after writing header + payload.
|
||||
*/
|
||||
/* ---- Write path ---- */
|
||||
|
||||
void ec_accumulate_data_block(aaruformat_context *ctx, const BlockHeader *block_header, const uint8_t *lzma_props,
|
||||
const uint8_t *payload, uint32_t payload_size, uint64_t file_offset);
|
||||
|
||||
/**
|
||||
* @brief Flush a completed data stripe slot: write parity blocks and record descriptor.
|
||||
*/
|
||||
void ec_flush_data_stripe(aaruformat_context *ctx, uint32_t slot);
|
||||
|
||||
/**
|
||||
* @brief Flush partial stripes, write ECMB and recovery footer.
|
||||
*
|
||||
* Called from aaruf_finalize_write() after index is written.
|
||||
*/
|
||||
void ec_finalize(aaruformat_context *ctx);
|
||||
|
||||
/* ---- Read path ---- */
|
||||
|
||||
/**
|
||||
* @brief Free all erasure coding state.
|
||||
* @brief Try to load the ECMB from the recovery footer at EOF.
|
||||
*
|
||||
* Called from aaruf_close().
|
||||
* Reads the last 160 bytes, checks for footerMagic, parses ECMB,
|
||||
* and populates the ec_read_* fields and block lookup hashmap.
|
||||
* Called from aaruf_open().
|
||||
*/
|
||||
void ec_load_ecmb(aaruformat_context *ctx);
|
||||
|
||||
/**
|
||||
* @brief Attempt to recover a data block that failed decompression or CRC verification.
|
||||
*
|
||||
* Looks up the block's stripe via the ECMB, reads surviving stripe members + parity,
|
||||
* RS-decodes the erased shard, and decompresses the recovered block.
|
||||
*
|
||||
* @param ctx Context with ec_recovery_available == true.
|
||||
* @param block_offset File offset of the corrupted block.
|
||||
* @param offset Sector offset within the block.
|
||||
* @param data Output buffer for the recovered sector.
|
||||
* @param length Output: bytes written to data.
|
||||
* @param sector_status Sector status from DDT.
|
||||
* @return AARUF_STATUS_OK on success, negative error code on failure.
|
||||
*/
|
||||
int32_t ec_recover_data_block(aaruformat_context *ctx, uint64_t block_offset, uint64_t offset,
|
||||
uint8_t *data, uint32_t *length, uint8_t sector_status);
|
||||
|
||||
/* ---- Cleanup ---- */
|
||||
|
||||
void ec_free(aaruformat_context *ctx);
|
||||
|
||||
#endif /* LIBAARUFORMAT_ERASURE_INTERNAL_H */
|
||||
|
||||
454
src/erasure.c
454
src/erasure.c
@@ -36,6 +36,27 @@
|
||||
#include "lib/gf256.h"
|
||||
#include "lib/reed_solomon.h"
|
||||
|
||||
/* =========================================================================
|
||||
* Read-path structures (not in on-disk format, internal only)
|
||||
* ========================================================================= */
|
||||
|
||||
/** @brief In-memory representation of one data stripe (parsed from ECMB). */
|
||||
typedef struct EcReadStripe
|
||||
{
|
||||
uint16_t actual_k; ///< Number of data blocks in this stripe.
|
||||
StripeDataBlockEntry *data_entries; ///< Array of actual_k entries.
|
||||
uint64_t *parity_offsets; ///< Array of M parity block file offsets.
|
||||
} EcReadStripe;
|
||||
|
||||
/** @brief Hash table entry mapping block file offset -> stripe index + position. */
|
||||
typedef struct EcBlockLookupEntry
|
||||
{
|
||||
uint64_t block_offset; ///< Key: file offset of the data block.
|
||||
uint32_t stripe_index; ///< Index into ec_read_stripes array.
|
||||
uint16_t position; ///< Position within the stripe (0..actual_k-1).
|
||||
UT_hash_handle hh;
|
||||
} EcBlockLookupEntry;
|
||||
|
||||
/* UT_array icd for completed stripe descriptors.
|
||||
* Each descriptor is a variable-length blob serialized in-place. We store
|
||||
* them as flat byte buffers since the size per stripe depends on K and M. */
|
||||
@@ -564,4 +585,437 @@ void ec_free(aaruformat_context *ctx)
|
||||
}
|
||||
|
||||
ctx->ec_enabled = false;
|
||||
|
||||
/* Free read-path state */
|
||||
if(ctx->ec_read_stripes)
|
||||
{
|
||||
EcReadStripe *stripes = (EcReadStripe *)ctx->ec_read_stripes;
|
||||
for(uint32_t i = 0; i < ctx->ec_read_stripe_count; i++)
|
||||
{
|
||||
free(stripes[i].data_entries);
|
||||
free(stripes[i].parity_offsets);
|
||||
}
|
||||
free(stripes);
|
||||
ctx->ec_read_stripes = NULL;
|
||||
}
|
||||
ctx->ec_read_stripe_count = 0;
|
||||
|
||||
/* Free block lookup hashmap */
|
||||
if(ctx->ec_block_lookup)
|
||||
{
|
||||
EcBlockLookupEntry *root = (EcBlockLookupEntry *)ctx->ec_block_lookup;
|
||||
EcBlockLookupEntry *entry, *tmp;
|
||||
HASH_ITER(hh, root, entry, tmp)
|
||||
{
|
||||
HASH_DEL(root, entry);
|
||||
free(entry);
|
||||
}
|
||||
ctx->ec_block_lookup = NULL;
|
||||
}
|
||||
|
||||
ctx->ec_recovery_available = false;
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* ECMB loading (read path)
|
||||
* ========================================================================= */
|
||||
|
||||
void ec_load_ecmb(aaruformat_context *ctx)
|
||||
{
|
||||
TRACE("Entering ec_load_ecmb(%p)", (void *)ctx);
|
||||
|
||||
/* Read recovery footer from last 160 bytes of file */
|
||||
aaruf_fseek(ctx->imageStream, 0, SEEK_END);
|
||||
int64_t file_size = aaruf_ftell(ctx->imageStream);
|
||||
if(file_size < (int64_t)sizeof(AaruRecoveryFooter))
|
||||
{
|
||||
TRACE("File too small for recovery footer");
|
||||
return;
|
||||
}
|
||||
|
||||
aaruf_fseek(ctx->imageStream, (aaru_off_t)(file_size - (int64_t)sizeof(AaruRecoveryFooter)), SEEK_SET);
|
||||
|
||||
AaruRecoveryFooter footer;
|
||||
if(fread(&footer, sizeof(AaruRecoveryFooter), 1, ctx->imageStream) != 1)
|
||||
{
|
||||
TRACE("Cannot read recovery footer");
|
||||
return;
|
||||
}
|
||||
|
||||
if(footer.footerMagic != AARU_RECOVERY_FOOTER_MAGIC)
|
||||
{
|
||||
TRACE("Recovery footer magic mismatch: 0x%016" PRIx64, footer.footerMagic);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Read ECMB header */
|
||||
aaruf_fseek(ctx->imageStream, (aaru_off_t)footer.ecmbOffset, SEEK_SET);
|
||||
|
||||
ErasureCodingMapHeader ecmb;
|
||||
if(fread(&ecmb, sizeof(ErasureCodingMapHeader), 1, ctx->imageStream) != 1)
|
||||
{
|
||||
TRACE("Cannot read ECMB header");
|
||||
return;
|
||||
}
|
||||
|
||||
if(ecmb.identifier != ErasureCodingMapBlock)
|
||||
{
|
||||
TRACE("ECMB identifier mismatch");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Read payload (uncompressed only for now) */
|
||||
if(ecmb.length == 0 || ecmb.length > 256 * 1024 * 1024) return; /* sanity limit */
|
||||
|
||||
uint8_t *payload = (uint8_t *)malloc((size_t)ecmb.length);
|
||||
if(!payload) return;
|
||||
|
||||
if(ecmb.compression == kCompressionNone)
|
||||
{
|
||||
if(fread(payload, (size_t)ecmb.cmpLength, 1, ctx->imageStream) != 1)
|
||||
{
|
||||
free(payload);
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Compressed ECMB payload — read compressed, decompress */
|
||||
uint8_t *cmp = (uint8_t *)malloc((size_t)ecmb.cmpLength);
|
||||
if(!cmp) { free(payload); return; }
|
||||
if(fread(cmp, (size_t)ecmb.cmpLength, 1, ctx->imageStream) != 1) { free(cmp); free(payload); return; }
|
||||
|
||||
if(ecmb.compression == kCompressionLzma)
|
||||
{
|
||||
size_t out_size = (size_t)ecmb.length;
|
||||
size_t lzma_src_size = (size_t)ecmb.cmpLength - LZMA_PROPERTIES_LENGTH;
|
||||
aaruf_lzma_decode_buffer(payload, &out_size, cmp + LZMA_PROPERTIES_LENGTH,
|
||||
&lzma_src_size, cmp, LZMA_PROPERTIES_LENGTH);
|
||||
}
|
||||
else if(ecmb.compression == kCompressionZstd)
|
||||
{
|
||||
aaruf_zstd_decode_buffer(payload, (size_t)ecmb.length, cmp, (size_t)ecmb.cmpLength);
|
||||
}
|
||||
free(cmp);
|
||||
}
|
||||
|
||||
/* Verify CRC64 */
|
||||
uint64_t computed_crc = aaruf_crc64_data(payload, (uint32_t)ecmb.length);
|
||||
if(computed_crc != ecmb.crc64)
|
||||
{
|
||||
TRACE("ECMB payload CRC64 mismatch");
|
||||
free(payload);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Parse stripe group descriptor */
|
||||
if(ecmb.length < sizeof(StripeGroupDescriptor)) { free(payload); return; }
|
||||
|
||||
StripeGroupDescriptor group;
|
||||
memcpy(&group, payload, sizeof(StripeGroupDescriptor));
|
||||
|
||||
ctx->ec_algorithm = ecmb.algorithm;
|
||||
ctx->ec_K = group.K;
|
||||
ctx->ec_M = group.M;
|
||||
ctx->ec_data_shard_size = group.shardSize;
|
||||
|
||||
/* Parse stripe descriptors */
|
||||
uint8_t *p = payload + sizeof(StripeGroupDescriptor);
|
||||
size_t remaining = (size_t)ecmb.length - sizeof(StripeGroupDescriptor);
|
||||
|
||||
EcReadStripe *stripes = (EcReadStripe *)calloc(group.stripeCount, sizeof(EcReadStripe));
|
||||
if(!stripes) { free(payload); return; }
|
||||
|
||||
EcBlockLookupEntry *lookup_root = NULL;
|
||||
|
||||
for(uint32_t s = 0; s < group.stripeCount; s++)
|
||||
{
|
||||
if(remaining < sizeof(uint16_t)) break;
|
||||
uint16_t ak;
|
||||
memcpy(&ak, p, sizeof(uint16_t)); p += sizeof(uint16_t); remaining -= sizeof(uint16_t);
|
||||
stripes[s].actual_k = ak;
|
||||
|
||||
size_t data_bytes = (size_t)ak * sizeof(StripeDataBlockEntry);
|
||||
size_t parity_bytes = (size_t)group.M * sizeof(StripeParityBlockEntry);
|
||||
if(remaining < data_bytes + parity_bytes) break;
|
||||
|
||||
stripes[s].data_entries = (StripeDataBlockEntry *)malloc(data_bytes);
|
||||
if(!stripes[s].data_entries) break;
|
||||
memcpy(stripes[s].data_entries, p, data_bytes); p += data_bytes; remaining -= data_bytes;
|
||||
|
||||
stripes[s].parity_offsets = (uint64_t *)malloc((size_t)group.M * sizeof(uint64_t));
|
||||
if(!stripes[s].parity_offsets) break;
|
||||
for(uint16_t m = 0; m < group.M; m++)
|
||||
{
|
||||
StripeParityBlockEntry pe;
|
||||
memcpy(&pe, p, sizeof(StripeParityBlockEntry)); p += sizeof(StripeParityBlockEntry);
|
||||
remaining -= sizeof(StripeParityBlockEntry);
|
||||
stripes[s].parity_offsets[m] = pe.offset;
|
||||
}
|
||||
|
||||
/* Build lookup hashmap entries for each data block in this stripe */
|
||||
for(uint16_t k = 0; k < ak; k++)
|
||||
{
|
||||
EcBlockLookupEntry *le = (EcBlockLookupEntry *)calloc(1, sizeof(EcBlockLookupEntry));
|
||||
if(!le) break;
|
||||
le->block_offset = stripes[s].data_entries[k].offset;
|
||||
le->stripe_index = s;
|
||||
le->position = k;
|
||||
HASH_ADD(hh, lookup_root, block_offset, sizeof(uint64_t), le);
|
||||
}
|
||||
}
|
||||
|
||||
free(payload);
|
||||
|
||||
ctx->ec_read_stripes = stripes;
|
||||
ctx->ec_read_stripe_count = group.stripeCount;
|
||||
ctx->ec_block_lookup = lookup_root;
|
||||
ctx->ec_recovery_available = true;
|
||||
|
||||
/* Create RS codec for decoding */
|
||||
if(!ctx->ec_rs_ctx)
|
||||
ctx->ec_rs_ctx = rs_create(group.K, group.M);
|
||||
|
||||
TRACE("ECMB loaded: K=%u M=%u shard_size=%u stripes=%u", group.K, group.M, group.shardSize, group.stripeCount);
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* Data block recovery (read path)
|
||||
* ========================================================================= */
|
||||
|
||||
int32_t ec_recover_data_block(aaruformat_context *ctx, uint64_t block_offset, uint64_t offset,
|
||||
uint8_t *data, uint32_t *length, uint8_t sector_status)
|
||||
{
|
||||
if(!ctx->ec_recovery_available || ctx->ec_recovery_in_progress) return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
|
||||
ctx->ec_recovery_in_progress = true;
|
||||
|
||||
/* Look up which stripe this block belongs to */
|
||||
EcBlockLookupEntry *le = NULL;
|
||||
HASH_FIND(hh, (EcBlockLookupEntry *)ctx->ec_block_lookup, &block_offset, sizeof(uint64_t), le);
|
||||
if(!le) { ctx->ec_recovery_in_progress = false; return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; }
|
||||
|
||||
uint32_t si = le->stripe_index;
|
||||
EcReadStripe *stripes = (EcReadStripe *)ctx->ec_read_stripes;
|
||||
EcReadStripe *stripe = &stripes[si];
|
||||
uint16_t K = ctx->ec_K;
|
||||
uint16_t M = ctx->ec_M;
|
||||
uint32_t shard_size = ctx->ec_data_shard_size;
|
||||
|
||||
/* Always use K+M shards for RS. For partial stripes (actual_k < K),
|
||||
* positions actual_k..K-1 are all-zero (calloc'd) and marked present.
|
||||
* This works because the encoding used the K-size generator matrix
|
||||
* and zero-contributions for unused positions. */
|
||||
uint16_t total_shards = K + M;
|
||||
|
||||
/* Allocate shard pointers and present flags */
|
||||
uint8_t **shards = (uint8_t **)calloc(total_shards, sizeof(uint8_t *));
|
||||
uint8_t *present = (uint8_t *)calloc(total_shards, 1);
|
||||
if(!shards || !present) { free(shards); free(present); ctx->ec_recovery_in_progress = false; return AARUF_ERROR_NOT_ENOUGH_MEMORY; }
|
||||
|
||||
for(uint16_t i = 0; i < total_shards; i++)
|
||||
{
|
||||
shards[i] = (uint8_t *)calloc(1, shard_size);
|
||||
if(!shards[i])
|
||||
{
|
||||
for(uint16_t j = 0; j < i; j++) free(shards[j]);
|
||||
free(shards); free(present);
|
||||
ctx->ec_recovery_in_progress = false;
|
||||
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
|
||||
}
|
||||
}
|
||||
|
||||
/* Read data shards from file and verify each one's CRC64 against ECMB */
|
||||
for(uint16_t k = 0; k < stripe->actual_k; k++)
|
||||
{
|
||||
StripeDataBlockEntry *de = &stripe->data_entries[k];
|
||||
uint32_t read_size = de->onDiskSize;
|
||||
if(read_size > shard_size) read_size = shard_size;
|
||||
|
||||
aaruf_fseek(ctx->imageStream, (aaru_off_t)de->offset, SEEK_SET);
|
||||
if(fread(shards[k], read_size, 1, ctx->imageStream) != 1)
|
||||
{
|
||||
present[k] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Verify CRC64 (zero-padded to shard_size via calloc) */
|
||||
uint64_t crc = aaruf_crc64_data(shards[k], shard_size);
|
||||
present[k] = (crc == de->shardCrc64) ? 1 : 0;
|
||||
}
|
||||
|
||||
/* Positions actual_k..K-1 are all-zero and present (unused stripe positions) */
|
||||
for(uint16_t k = stripe->actual_k; k < K; k++)
|
||||
present[k] = 1; /* All-zero shards, implicitly correct */
|
||||
|
||||
/* Read parity shards */
|
||||
for(uint16_t m = 0; m < M; m++)
|
||||
{
|
||||
uint16_t shard_idx = K + m;
|
||||
uint64_t parity_offset = stripe->parity_offsets[m];
|
||||
|
||||
aaruf_fseek(ctx->imageStream, (aaru_off_t)parity_offset, SEEK_SET);
|
||||
|
||||
/* Read parity block header */
|
||||
BlockHeader parity_header;
|
||||
if(fread(&parity_header, sizeof(BlockHeader), 1, ctx->imageStream) != 1)
|
||||
{
|
||||
present[shard_idx] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Read and decompress parity payload */
|
||||
if(parity_header.compression == kCompressionNone)
|
||||
{
|
||||
uint32_t to_read = parity_header.length;
|
||||
if(to_read > shard_size) to_read = shard_size;
|
||||
if(fread(shards[shard_idx], to_read, 1, ctx->imageStream) != 1)
|
||||
{
|
||||
present[shard_idx] = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
else if(parity_header.compression == kCompressionLzma)
|
||||
{
|
||||
uint32_t cmp_data_len = parity_header.cmpLength - LZMA_PROPERTIES_LENGTH;
|
||||
uint8_t lzma_props[LZMA_PROPERTIES_LENGTH];
|
||||
if(fread(lzma_props, LZMA_PROPERTIES_LENGTH, 1, ctx->imageStream) != 1 )
|
||||
{
|
||||
present[shard_idx] = 0;
|
||||
continue;
|
||||
}
|
||||
uint8_t *cmp = (uint8_t *)malloc(cmp_data_len);
|
||||
if(!cmp) { present[shard_idx] = 0; continue; }
|
||||
if(fread(cmp, cmp_data_len, 1, ctx->imageStream) != 1) { free(cmp); present[shard_idx] = 0; continue; }
|
||||
|
||||
size_t out_size = shard_size;
|
||||
size_t lzma_src = (size_t)cmp_data_len;
|
||||
aaruf_lzma_decode_buffer(shards[shard_idx], &out_size, cmp, &lzma_src, lzma_props, LZMA_PROPERTIES_LENGTH);
|
||||
free(cmp);
|
||||
}
|
||||
else if(parity_header.compression == kCompressionZstd)
|
||||
{
|
||||
uint8_t *cmp = (uint8_t *)malloc(parity_header.cmpLength);
|
||||
if(!cmp) { present[shard_idx] = 0; continue; }
|
||||
if(fread(cmp, parity_header.cmpLength, 1, ctx->imageStream) != 1) { free(cmp); present[shard_idx] = 0; continue; }
|
||||
|
||||
aaruf_zstd_decode_buffer(shards[shard_idx], shard_size, cmp, parity_header.cmpLength);
|
||||
free(cmp);
|
||||
}
|
||||
else
|
||||
{
|
||||
present[shard_idx] = 0;
|
||||
continue;
|
||||
}
|
||||
present[shard_idx] = 1;
|
||||
}
|
||||
|
||||
/* Always use the original RS(K,M) codec — partial stripes have zero-padded unused positions */
|
||||
rs_context *rs = (rs_context *)ctx->ec_rs_ctx;
|
||||
if(!rs)
|
||||
{
|
||||
for(uint16_t i = 0; i < total_shards; i++) free(shards[i]);
|
||||
free(shards); free(present);
|
||||
ctx->ec_recovery_in_progress = false;
|
||||
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
|
||||
}
|
||||
|
||||
/* RS decode */
|
||||
int rc = rs_decode(rs, shards, present, shard_size);
|
||||
|
||||
if(rc != 0)
|
||||
{
|
||||
for(uint16_t i = 0; i < total_shards; i++) free(shards[i]);
|
||||
free(shards); free(present);
|
||||
ctx->ec_recovery_in_progress = false;
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
|
||||
/* Find the shard corresponding to our corrupted block */
|
||||
uint16_t our_pos = le->position;
|
||||
uint8_t *recovered_shard = shards[our_pos];
|
||||
|
||||
/* Parse the recovered BlockHeader */
|
||||
BlockHeader recovered_header;
|
||||
memcpy(&recovered_header, recovered_shard, sizeof(BlockHeader));
|
||||
|
||||
/* Decompress the recovered payload */
|
||||
uint32_t hdr_size = sizeof(BlockHeader);
|
||||
uint8_t *recovered_payload = recovered_shard + hdr_size;
|
||||
uint32_t payload_len = recovered_header.cmpLength;
|
||||
|
||||
uint8_t *block = NULL;
|
||||
|
||||
if(recovered_header.compression == kCompressionNone)
|
||||
{
|
||||
block = (uint8_t *)malloc(recovered_header.length);
|
||||
if(block) memcpy(block, recovered_payload, recovered_header.length);
|
||||
}
|
||||
else if(recovered_header.compression == kCompressionLzma)
|
||||
{
|
||||
uint8_t *lzma_props = recovered_payload;
|
||||
uint8_t *lzma_data = recovered_payload + LZMA_PROPERTIES_LENGTH;
|
||||
uint32_t lzma_data_len = payload_len - LZMA_PROPERTIES_LENGTH;
|
||||
|
||||
block = (uint8_t *)malloc(recovered_header.length);
|
||||
if(block)
|
||||
{
|
||||
size_t out_size = recovered_header.length;
|
||||
size_t lzma_src2 = (size_t)lzma_data_len;
|
||||
aaruf_lzma_decode_buffer(block, &out_size, lzma_data, &lzma_src2, lzma_props, LZMA_PROPERTIES_LENGTH);
|
||||
}
|
||||
}
|
||||
else if(recovered_header.compression == kCompressionZstd)
|
||||
{
|
||||
block = (uint8_t *)malloc(recovered_header.length);
|
||||
if(block)
|
||||
aaruf_zstd_decode_buffer(block, recovered_header.length, recovered_payload, payload_len);
|
||||
}
|
||||
else if(recovered_header.compression == kCompressionFlac)
|
||||
{
|
||||
block = (uint8_t *)malloc(recovered_header.length);
|
||||
if(block)
|
||||
aaruf_flac_decode_redbook_buffer(block, recovered_header.length, recovered_payload, payload_len);
|
||||
}
|
||||
|
||||
int32_t result = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
|
||||
if(block)
|
||||
{
|
||||
/* Verify recovered uncompressed data CRC64 */
|
||||
uint64_t block_crc = aaruf_crc64_data(block, recovered_header.length);
|
||||
if(block_crc == recovered_header.crc64)
|
||||
{
|
||||
/* Extract the requested sector */
|
||||
uint32_t sector_size = recovered_header.sectorSize;
|
||||
if(sector_size > 0 && offset * sector_size + sector_size <= recovered_header.length)
|
||||
{
|
||||
memcpy(data, block + offset * sector_size, sector_size);
|
||||
*length = sector_size;
|
||||
result = AARUF_STATUS_OK;
|
||||
|
||||
/* Cache the recovered block so subsequent sector reads from the same
|
||||
* block don't re-trigger recovery (this is the critical optimization). */
|
||||
add_to_cache_uint64(&ctx->block_cache, block_offset, block);
|
||||
|
||||
/* Also cache the recovered BlockHeader */
|
||||
BlockHeader *cached_hdr = (BlockHeader *)malloc(sizeof(BlockHeader));
|
||||
if(cached_hdr)
|
||||
{
|
||||
memcpy(cached_hdr, &recovered_header, sizeof(BlockHeader));
|
||||
add_to_cache_uint64(&ctx->block_header_cache, block_offset, cached_hdr);
|
||||
}
|
||||
|
||||
block = NULL; /* Ownership transferred to cache — don't free */
|
||||
}
|
||||
}
|
||||
free(block); /* Only frees if not transferred to cache */
|
||||
}
|
||||
|
||||
for(uint16_t i = 0; i < total_shards; i++) free(shards[i]);
|
||||
free(shards);
|
||||
free(present);
|
||||
|
||||
ctx->ec_recovery_in_progress = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
|
||||
#include <aaruformat.h>
|
||||
|
||||
#include "erasure_internal.h"
|
||||
#include "internal.h"
|
||||
#include "log.h"
|
||||
#include "utarray.h"
|
||||
@@ -684,6 +685,9 @@ AARU_EXPORT void *AARU_CALL aaruf_open(const char *filepath, const bool resume_m
|
||||
ctx->library_major_version = LIBAARUFORMAT_MAJOR_VERSION;
|
||||
ctx->library_minor_version = LIBAARUFORMAT_MINOR_VERSION;
|
||||
|
||||
/* Try to load erasure coding recovery metadata from EOF footer */
|
||||
ec_load_ecmb(ctx);
|
||||
|
||||
if(!resume_mode)
|
||||
{
|
||||
TRACE("Exiting aaruf_open() = %p", ctx);
|
||||
|
||||
43
src/read.c
43
src/read.c
@@ -21,6 +21,7 @@
|
||||
|
||||
#include <aaruformat.h>
|
||||
|
||||
#include "erasure_internal.h"
|
||||
#include "internal.h"
|
||||
#include "log.h"
|
||||
#include "ngcw/lfg.h"
|
||||
@@ -818,7 +819,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
block_header->length);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
lzma_size = block_header->cmpLength - LZMA_PROPERTIES_LENGTH;
|
||||
@@ -853,7 +854,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(cmp_data);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
read_bytes = fread(cmp_data, 1, lzma_size, ctx->imageStream);
|
||||
@@ -864,7 +865,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(block);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
TRACE("Decompressing block of size %zu bytes", block_header->length);
|
||||
@@ -879,7 +880,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(block);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
if(read_bytes != block_header->length)
|
||||
@@ -889,7 +890,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(block);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
free(cmp_data);
|
||||
@@ -901,7 +902,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
FATAL("Invalid zstd block lengths (cmpLength=%u, length=%u)", block_header->cmpLength,
|
||||
block_header->length);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
TRACE("Allocating memory for block of size %zu bytes", block_header->length);
|
||||
@@ -932,7 +933,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(cmp_data);
|
||||
free(block);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
read_bytes = aaruf_zstd_decode_buffer(block, block_header->length, cmp_data, block_header->cmpLength);
|
||||
@@ -942,7 +943,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(cmp_data);
|
||||
free(block);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
free(cmp_data);
|
||||
@@ -980,7 +981,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(block);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
TRACE("Decompressing block of size %zu bytes", block_header->length);
|
||||
@@ -994,7 +995,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
free(block);
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
free(cmp_data);
|
||||
@@ -1003,7 +1004,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
default:
|
||||
FATAL("Unsupported compression %d", block_header->compression);
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_UNSUPPORTED_COMPRESSION");
|
||||
return AARUF_ERROR_UNSUPPORTED_COMPRESSION;
|
||||
goto ec_try_recovery;
|
||||
}
|
||||
|
||||
// Add block to cache
|
||||
@@ -1166,6 +1167,26 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, const uint64_t se
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_STATUS_OK");
|
||||
return AARUF_STATUS_OK;
|
||||
|
||||
/* Erasure coding recovery: attempt to reconstruct a block that failed
|
||||
* decompression due to on-disk corruption. ec_recover_data_block()
|
||||
* reads the stripe, RS-decodes the erased shard, decompresses it,
|
||||
* and extracts the requested sector. */
|
||||
ec_try_recovery:
|
||||
if(ctx->ec_recovery_available && !ctx->ec_recovery_in_progress)
|
||||
{
|
||||
TRACE("Attempting erasure coding recovery for block at offset %" PRIu64, block_offset);
|
||||
int32_t rc = ec_recover_data_block(ctx, block_offset, offset, data, length, *sector_status);
|
||||
if(rc == AARUF_STATUS_OK)
|
||||
{
|
||||
TRACE("Erasure coding recovery succeeded");
|
||||
return AARUF_STATUS_OK;
|
||||
}
|
||||
TRACE("Erasure coding recovery failed: %d", rc);
|
||||
}
|
||||
|
||||
TRACE("Exiting aaruf_read_sector() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK");
|
||||
return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user