erasure_internal.h File Reference

#include "aaruformat/context.h"
#include "aaruformat/structs/data.h"

Go to the source code of this file.

Functions
void	ec_accumulate_data_block (aaruformat_context *ctx, const BlockHeader *block_header, const uint8_t *lzma_props, const uint8_t *payload, uint32_t payload_size, uint64_t file_offset)
	Accumulate parity for a data block that was just written to disk.
void	ec_flush_data_stripe (aaruformat_context *ctx, uint32_t slot)
	Write M parity blocks for a completed data stripe slot and record the stripe descriptor.
void	ec_finalize (aaruformat_context *ctx)
	Flush all partial data stripes and write parity for all groups + ECMB + recovery footer.
void	ec_load_ecmb (aaruformat_context *ctx)
	Try to load the ECMB from the recovery footer at EOF.
int32_t	ec_recover_data_block (aaruformat_context *ctx, uint64_t block_offset, uint64_t offset, uint8_t *data, uint32_t *length, uint8_t sector_status)
	Attempt to recover a data block that failed decompression or CRC verification.
int32_t	ec_recover_raw_block (aaruformat_context *ctx, uint64_t block_offset, uint8_t **recovered_data, uint32_t *recovered_size, void *stripes, uint32_t stripe_count, void *lookup, uint16_t group_K, uint16_t group_M, uint32_t group_shard_size)
	Attempt to recover a raw block at a given file offset using a specified stripe group.
int32_t	ec_recover_meta_block (aaruformat_context *ctx, uint64_t block_offset, uint8_t **recovered_data, uint32_t *recovered_size)
	Attempt to recover a metadata/media-tag block at a given file offset.
int32_t	ec_recover_ddt_block (aaruformat_context *ctx, uint64_t block_offset, uint8_t **recovered_data, uint32_t *recovered_size)
	Attempt to recover a DDT secondary block at a given file offset.
void	ec_free (aaruformat_context *ctx)
	Free all erasure coding state from the context.

Function Documentation

ec_accumulate_data_block()

void ec_accumulate_data_block	(	aaruformat_context *	ctx,
		const BlockHeader *	block_header,
		const uint8_t *	lzma_props,
		const uint8_t *	payload,
		uint32_t	payload_size,
		uint64_t	file_offset )

Accumulate parity for a data block that was just written to disk.

Called from aaruf_close_current_block() after writing BlockHeader + payload to the file but before freeing the compressed data buffer.

Parameters

ctx	Context with EC enabled.
block_header	Pointer to the BlockHeader that was written.
lzma_props	LZMA properties (may be NULL if not LZMA).
payload	Compressed payload data (or uncompressed if no compression).
payload_size	Size of the payload data.
file_offset	File offset where the block was written.

Definition at line 209 of file erasure.c.

References aaruf_crc64_data(), BlockHeader::compression, ec_accumulate_data_block(), aaruformat_context::ec_data_block_offsets, aaruformat_context::ec_data_block_sizes, aaruformat_context::ec_data_parity, aaruformat_context::ec_data_shard_crcs, aaruformat_context::ec_data_shard_size, aaruformat_context::ec_data_stripe_counts, aaruformat_context::ec_enabled, ec_flush_data_stripe(), aaruformat_context::ec_K, aaruformat_context::ec_M, aaruformat_context::ec_rs_ctx, aaruformat_context::ec_total_data_blocks, kCompressionLzma, LZMA_PROPERTIES_LENGTH, rs_encode_incremental(), rs_get_coefficient(), and TRACE.

Referenced by aaruf_close_current_block(), and ec_accumulate_data_block().

ec_finalize()

void ec_finalize

(

aaruformat_context *

ctx

)

Flush all partial data stripes and write parity for all groups + ECMB + recovery footer.

Definition at line 585 of file erasure.c.

References AARU_RECOVERY_FOOTER_MAGIC, aaruf_crc64_data(), aaruf_fseek(), aaruf_ftell(), ErasureCodingMapHeader::algorithm, AaruRecoveryFooter::backupHeader, DdtHeader2::blockAlignmentShift, IndexEntry::blockType, ErasureCodingMapHeader::cmpCrc64, BlockHeader::cmpLength, ErasureCodingMapHeader::cmpLength, ErasureCodingMapHeader::compression, ErasureCodingMapHeader::crc64, DataBlock, IndexEntry::dataType, DeDuplicationTable, DeDuplicationTable2, DeDuplicationTableSecondary, aaruformat_context::ec_algorithm, ec_collect_blocks_by_type(), aaruformat_context::ec_data_shard_size, aaruformat_context::ec_data_stripe_counts, aaruformat_context::ec_data_stripes, aaruformat_context::ec_enabled, ec_finalize(), ec_flush_data_stripe(), aaruformat_context::ec_K, aaruformat_context::ec_M, ec_write_batch_parity(), AaruRecoveryFooter::ecmbLength, AaruRecoveryFooter::ecmbOffset, IndexHeader3::entries, ErasureCodingMapBlock, AaruRecoveryFooter::footerMagic, StripeGroupDescriptor::groupType, aaruformat_context::header, AaruRecoveryFooter::headerCrc64, ErasureCodingMapHeader::identifier, aaruformat_context::imageStream, aaruformat_context::index_entries, IndexBlock, IndexBlock2, IndexBlock3, AaruHeaderV2::indexOffset, StripeGroupDescriptor::interleaveDepth, StripeGroupDescriptor::K, kCompressionNone, kDataTypeErasureParity, kDataTypeErasureParityDdt, kDataTypeErasureParityDdtPrimary, kDataTypeErasureParityIndex, kDataTypeErasureParityMeta, kDataTypeUserData, kECGroupData, kECGroupDdtPrimary, kECGroupDdtSecondary, kECGroupIndex, kECGroupMetadata, ErasureCodingMapHeader::length, StripeGroupDescriptor::M, IndexEntry::offset, StripeGroupDescriptor::shardSize, StripeGroupDescriptor::stripeCount, ErasureCodingMapHeader::stripeGroupCount, TRACE, aaruformat_context::user_data_ddt_header, and WRITE_GROUP.

Referenced by aaruf_finalize_write(), and ec_finalize().

ec_flush_data_stripe()

void ec_flush_data_stripe	(	aaruformat_context *	ctx,
		uint32_t	slot )

Write M parity blocks for a completed data stripe slot and record the stripe descriptor.

Parameters

ctx	Context with EC enabled.
slot	Stripe slot index (0 .. K-1).

Definition at line 323 of file erasure.c.

References aaruf_crc64_data(), aaruf_fseek(), aaruf_ftell(), DdtHeader2::blockAlignmentShift, BlockHeader::cmpCrc64, BlockHeader::cmpLength, BlockHeader::compression, BlockHeader::crc64, DataBlock, aaruformat_context::dirty_index_block, aaruformat_context::ec_data_block_offsets, aaruformat_context::ec_data_block_sizes, aaruformat_context::ec_data_parity, aaruformat_context::ec_data_shard_crcs, aaruformat_context::ec_data_shard_size, aaruformat_context::ec_data_stripe_counts, aaruformat_context::ec_data_stripes, ec_flush_data_stripe(), aaruformat_context::ec_K, aaruformat_context::ec_M, BlockHeader::identifier, aaruformat_context::imageStream, aaruformat_context::index_entries, kCompressionNone, kDataTypeErasureParity, BlockHeader::length, aaruformat_context::next_block_position, StripeDataBlockEntry::offset, StripeParityBlockEntry::offset, StripeDataBlockEntry::onDiskSize, BlockHeader::sectorSize, StripeDataBlockEntry::shardCrc64, BlockHeader::type, and aaruformat_context::user_data_ddt_header.

Referenced by ec_accumulate_data_block(), ec_finalize(), and ec_flush_data_stripe().

ec_free()

void ec_free

(

aaruformat_context *

ctx

)

Free all erasure coding state from the context.

Called from aaruf_close().

Parameters

ctx

Context.

Definition at line 934 of file erasure.c.

References aaruformat_context::ec_block_lookup, aaruformat_context::ec_data_block_offsets, aaruformat_context::ec_data_block_sizes, aaruformat_context::ec_data_parity, aaruformat_context::ec_data_shard_crcs, aaruformat_context::ec_data_stripe_counts, aaruformat_context::ec_data_stripes, aaruformat_context::ec_ddt_block_lookup, aaruformat_context::ec_ddt_stripe_count, aaruformat_context::ec_ddt_stripes, aaruformat_context::ec_enabled, ec_free(), aaruformat_context::ec_K, aaruformat_context::ec_M, aaruformat_context::ec_meta_block_lookup, aaruformat_context::ec_meta_stripe_count, aaruformat_context::ec_meta_stripes, aaruformat_context::ec_read_stripe_count, aaruformat_context::ec_read_stripes, aaruformat_context::ec_recovery_available, aaruformat_context::ec_rs_ctx, and rs_free().

Referenced by aaruf_close(), and ec_free().

ec_load_ecmb()

void ec_load_ecmb

(

aaruformat_context *

ctx

)

Try to load the ECMB from the recovery footer at EOF.

Reads the last 160 bytes, checks for footerMagic, parses ECMB, and populates the ec_read_* fields and block lookup hashmap. Called from aaruf_open().

Definition at line 1046 of file erasure.c.

References AARU_RECOVERY_FOOTER_MAGIC, aaruf_crc64_data(), aaruf_fseek(), aaruf_ftell(), aaruf_lzma_decode_buffer(), aaruf_zstd_decode_buffer(), EcReadStripe::actual_k, ErasureCodingMapHeader::algorithm, EcBlockLookupEntry::block_offset, ErasureCodingMapHeader::cmpLength, ErasureCodingMapHeader::compression, ErasureCodingMapHeader::crc64, EcReadStripe::data_entries, aaruformat_context::ec_algorithm, aaruformat_context::ec_block_lookup, aaruformat_context::ec_data_shard_size, aaruformat_context::ec_ddt_block_lookup, aaruformat_context::ec_ddt_K, aaruformat_context::ec_ddt_M, aaruformat_context::ec_ddt_shard_size, aaruformat_context::ec_ddt_stripe_count, aaruformat_context::ec_ddt_stripes, aaruformat_context::ec_K, ec_load_ecmb(), aaruformat_context::ec_M, aaruformat_context::ec_meta_block_lookup, aaruformat_context::ec_meta_K, aaruformat_context::ec_meta_M, aaruformat_context::ec_meta_shard_size, aaruformat_context::ec_meta_stripe_count, aaruformat_context::ec_meta_stripes, aaruformat_context::ec_read_stripe_count, aaruformat_context::ec_read_stripes, aaruformat_context::ec_recovery_available, aaruformat_context::ec_rs_ctx, AaruRecoveryFooter::ecmbOffset, ErasureCodingMapBlock, AaruRecoveryFooter::footerMagic, StripeGroupDescriptor::groupType, ErasureCodingMapHeader::identifier, aaruformat_context::imageStream, StripeGroupDescriptor::K, kCompressionLzma, kCompressionNone, kCompressionZstd, kECGroupData, kECGroupDdtSecondary, kECGroupMetadata, ErasureCodingMapHeader::length, LZMA_PROPERTIES_LENGTH, StripeGroupDescriptor::M, StripeDataBlockEntry::offset, StripeParityBlockEntry::offset, EcReadStripe::parity_offsets, EcBlockLookupEntry::position, rs_create(), StripeGroupDescriptor::shardSize, EcBlockLookupEntry::stripe_index, StripeGroupDescriptor::stripeCount, ErasureCodingMapHeader::stripeGroupCount, and TRACE.

Referenced by aaruf_open(), and ec_load_ecmb().

ec_recover_data_block()

int32_t ec_recover_data_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint64_t	offset,
		uint8_t *	data,
		uint32_t *	length,
		uint8_t	sector_status )

Attempt to recover a data block that failed decompression or CRC verification.

Looks up the block's stripe via the ECMB, reads surviving stripe members + parity, RS-decodes the erased shard, and decompresses the recovered block.

Parameters

ctx	Context with ec_recovery_available == true.
block_offset	File offset of the corrupted block.
offset	Sector offset within the block.
data	Output buffer for the recovered sector.
length	Output: bytes written to data.
sector_status	Sector status from DDT.

Returns

AARUF_STATUS_OK on success, negative error code on failure.

Definition at line 1303 of file erasure.c.

References aaruf_crc64_data(), AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK, AARUF_ERROR_NOT_ENOUGH_MEMORY, aaruf_flac_decode_redbook_buffer(), aaruf_fseek(), aaruf_lzma_decode_buffer(), AARUF_STATUS_OK, aaruf_zstd_decode_buffer(), EcReadStripe::actual_k, add_to_cache_uint64(), aaruformat_context::block_cache, aaruformat_context::block_header_cache, BlockHeader::cmpLength, BlockHeader::compression, BlockHeader::crc64, EcReadStripe::data_entries, aaruformat_context::ec_block_lookup, aaruformat_context::ec_data_shard_size, aaruformat_context::ec_K, aaruformat_context::ec_M, aaruformat_context::ec_read_stripes, ec_recover_data_block(), aaruformat_context::ec_recovery_available, aaruformat_context::ec_recovery_in_progress, aaruformat_context::ec_rs_ctx, aaruformat_context::imageStream, kCompressionFlac, kCompressionLzma, kCompressionNone, kCompressionZstd, BlockHeader::length, LZMA_PROPERTIES_LENGTH, StripeDataBlockEntry::offset, StripeDataBlockEntry::onDiskSize, EcReadStripe::parity_offsets, EcBlockLookupEntry::position, rs_decode(), BlockHeader::sectorSize, StripeDataBlockEntry::shardCrc64, and EcBlockLookupEntry::stripe_index.

Referenced by aaruf_read_sector(), and ec_recover_data_block().

ec_recover_ddt_block()

int32_t ec_recover_ddt_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size )

Attempt to recover a DDT secondary block at a given file offset.

Definition at line 918 of file erasure.c.

References AARUF_ERROR_CANNOT_READ_BLOCK, aaruformat_context::ec_ddt_block_lookup, aaruformat_context::ec_ddt_K, aaruformat_context::ec_ddt_M, aaruformat_context::ec_ddt_shard_size, aaruformat_context::ec_ddt_stripe_count, aaruformat_context::ec_ddt_stripes, ec_recover_ddt_block(), and ec_recover_raw_block().

Referenced by decode_ddt_entry_v2(), and ec_recover_ddt_block().

ec_recover_meta_block()

int32_t ec_recover_meta_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size )

Attempt to recover a metadata/media-tag block at a given file offset.

Convenience wrapper around ec_recover_raw_block using the metadata group.

Definition at line 909 of file erasure.c.

References AARUF_ERROR_CANNOT_READ_BLOCK, aaruformat_context::ec_meta_block_lookup, aaruformat_context::ec_meta_K, aaruformat_context::ec_meta_M, aaruformat_context::ec_meta_shard_size, aaruformat_context::ec_meta_stripe_count, aaruformat_context::ec_meta_stripes, ec_recover_meta_block(), and ec_recover_raw_block().

Referenced by ec_recover_meta_block(), process_data_block(), process_tracks_block(), and recover_dpm_block().

ec_recover_raw_block()

int32_t ec_recover_raw_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size,
		void *	stripes,
		uint32_t	stripe_count,
		void *	lookup,
		uint16_t	group_K,
		uint16_t	group_M,
		uint32_t	group_shard_size )

Attempt to recover a raw block at a given file offset using a specified stripe group.

Reads surviving stripe members + parity, RS-decodes the erased shard, returns the recovered raw on-disk bytes.

Parameters

ctx	Context.
block_offset	File offset of the corrupted block.
recovered_data	Output: malloc'd buffer with recovered bytes (caller frees). NULL on failure.
recovered_size	Output: size of recovered data.
stripes	EcReadStripe array for the group.
stripe_count	Number of stripes.
lookup	Block lookup hashmap for the group.
group_K	K for this group.
group_M	M for this group.
group_shard_size	Shard size for this group.

Returns

0 on success, negative on failure.

Definition at line 827 of file erasure.c.

References aaruf_crc64_data(), AARUF_ERROR_CANNOT_READ_BLOCK, AARUF_ERROR_NOT_ENOUGH_MEMORY, aaruf_fseek(), AARUF_STATUS_OK, EcReadStripe::actual_k, EcReadStripe::data_entries, ec_recover_raw_block(), aaruformat_context::ec_recovery_available, aaruformat_context::ec_recovery_in_progress, aaruformat_context::imageStream, BlockHeader::length, StripeDataBlockEntry::offset, StripeDataBlockEntry::onDiskSize, EcReadStripe::parity_offsets, EcBlockLookupEntry::position, rs_create(), rs_decode(), rs_free(), StripeDataBlockEntry::shardCrc64, and EcBlockLookupEntry::stripe_index.

Referenced by ec_recover_ddt_block(), ec_recover_meta_block(), and ec_recover_raw_block().