Erasure coding setup, parity accumulation, and write-path integration. More...

#include <stdlib.h>
#include <string.h>
#include "aaruformat.h"
#include "aaruformat/context.h"
#include "aaruformat/consts.h"
#include "aaruformat/enums.h"
#include "aaruformat/errors.h"
#include "aaruformat/structs/data.h"
#include "aaruformat/structs/ddt.h"
#include "aaruformat/structs/erasure.h"
#include "aaruformat/structs/index.h"
#include "internal.h"
#include "log.h"
#include "lib/gf256.h"
#include "lib/reed_solomon.h"

Data Structures
struct	EcReadStripe
	In-memory representation of one data stripe (parsed from ECMB). More...
struct	EcBlockLookupEntry
	Hash table entry mapping block file offset -> stripe index + position. More...

Macros
#define	WRITE_GROUP(desc, desc_len, grp)

Typedefs
typedef struct EcReadStripe	EcReadStripe
	In-memory representation of one data stripe (parsed from ECMB).
typedef struct EcBlockLookupEntry	EcBlockLookupEntry
	Hash table entry mapping block file offset -> stripe index + position.

Functions
void	ec_flush_data_stripe (aaruformat_context *ctx, uint32_t slot)
	Write M parity blocks for a completed data stripe slot and record the stripe descriptor.
int32_t	aaruf_set_erasure_coding (void *context, uint8_t algorithm, uint16_t K, uint16_t M)
	Configure erasure coding for a newly created image.
int32_t	aaruf_set_erasure_coding_auto (void *context, uint8_t recovery_percent)
	Configure erasure coding from a desired recovery percentage.
void	ec_accumulate_data_block (aaruformat_context ctx, const BlockHeader block_header, const uint8_t lzma_props, const uint8_t payload, uint32_t payload_size, uint64_t file_offset)
	Accumulate parity for a data block that was just written to disk.
static void	ec_write_batch_parity (aaruformat_context ctx, const uint64_t offsets, const uint32_t sizes, uint32_t block_count, uint8_t group_type, uint16_t parity_data_type, uint8_t out_desc, size_t out_desc_len, StripeGroupDescriptor *out_group)
	Compute and write batch parity for a set of blocks already on disk.
static void	ec_collect_blocks_by_type (aaruformat_context ctx, uint32_t block_type, uint64_t out_offsets, uint32_t out_sizes, uint32_t out_count)
	Collect file offsets and on-disk sizes for index entries matching a block type.
void	ec_finalize (aaruformat_context *ctx)
	Flush all partial data stripes and write parity for all groups + ECMB + recovery footer.
int32_t	ec_recover_raw_block (aaruformat_context ctx, uint64_t block_offset, uint8_t recovered_data, uint32_t recovered_size, void stripes_ptr, uint32_t stripe_count, void lookup_ptr, uint16_t group_K, uint16_t group_M, uint32_t group_shard_size)
	Attempt to recover a raw block at a given file offset using a specified stripe group.
int32_t	ec_recover_meta_block (aaruformat_context ctx, uint64_t block_offset, uint8_t recovered_data, uint32_t recovered_size)
	Attempt to recover a metadata/media-tag block at a given file offset.
int32_t	ec_recover_ddt_block (aaruformat_context ctx, uint64_t block_offset, uint8_t recovered_data, uint32_t recovered_size)
	Attempt to recover a DDT secondary block at a given file offset.
void	ec_free (aaruformat_context *ctx)
	Free all erasure coding state from the context.
void	ec_load_ecmb (aaruformat_context *ctx)
	Try to load the ECMB from the recovery footer at EOF.
int32_t	ec_recover_data_block (aaruformat_context ctx, uint64_t block_offset, uint64_t offset, uint8_t data, uint32_t *length, uint8_t sector_status)
	Attempt to recover a data block that failed decompression or CRC verification.

Variables
static UT_icd	ec_stripe_icd = {sizeof(uint8_t), NULL, NULL, NULL}

Detailed Description

Erasure coding setup, parity accumulation, and write-path integration.

Definition in file erasure.c.

Macro Definition Documentation

◆ WRITE_GROUP

#define WRITE_GROUP	(	desc,
		desc_len,
		grp )

Value:

if(desc) { memcpy(wp, &(grp), sizeof(StripeGroupDescriptor)); wp += sizeof(StripeGroupDescriptor); \

memcpy(wp, (desc), (desc_len)); wp += (desc_len); }

StripeGroupDescriptor

Describes one protection group within the ECMB payload.

Definition erasure.h:69

Referenced by ec_finalize().

Typedef Documentation

◆ EcBlockLookupEntry

typedef struct EcBlockLookupEntry EcBlockLookupEntry

Hash table entry mapping block file offset -> stripe index + position.

◆ EcReadStripe

typedef struct EcReadStripe EcReadStripe

In-memory representation of one data stripe (parsed from ECMB).

Function Documentation

◆ aaruf_set_erasure_coding()

int32_t aaruf_set_erasure_coding	(	void *	context,
		uint8_t	algorithm,
		uint16_t	K,
		uint16_t	M )

Configure erasure coding for a newly created image.

Must be called after aaruf_create() and before the first aaruf_write_sector(). Allocates K interleaved stripe slots, each with M parity buffers.

Parameters

context	Opaque context from aaruf_create().
algorithm	ErasureCodingAlgorithm (0=XOR, 1=RS).
K	Data blocks per stripe (>= 1).
M	Parity blocks per stripe (>= 1).

Returns: AARUF_STATUS_OK on success, error code otherwise.

Definition at line 82 of file erasure.c.

Referenced by aaruf_set_erasure_coding(), and aaruf_set_erasure_coding_auto().

◆ aaruf_set_erasure_coding_auto()

int32_t aaruf_set_erasure_coding_auto	(	void *	context,
		uint8_t	recovery_percent )

Configure erasure coding from a desired recovery percentage.

Computes K and M from the requested percentage of data that should be recoverable. Higher recovery percentages produce higher M values (more parity blocks = better burst-corruption tolerance).

Strategy: M = max(2, min(8, round(20 * percent / 100))) K = M * 100 / percent

This keeps K roughly constant (~16–40) while scaling M monotonically with the requested recovery level:

1% → RS(200, 2) M=2, minimum burst tolerance 5% → RS(40, 2) M=2 10% → RS(20, 2) M=2 15% → RS(20, 3) M=3, burst tolerance grows 25% → RS(20, 5) M=5 50% → RS(16, 8) M=8 (capped), strong burst tolerance 100% → RS(8, 8) M=8 (capped), maximum protection

Parameters

context	Opaque context from aaruf_create().
recovery_percent	Desired recoverable percentage (1..100).

Returns: AARUF_STATUS_OK on success, error code otherwise.

Definition at line 174 of file erasure.c.

References AARU_CALL, AARU_EXPORT, AARUF_ERROR_INCORRECT_DATA_SIZE, aaruf_set_erasure_coding(), aaruf_set_erasure_coding_auto(), kErasureCodingRsVandermonde, and TRACE.

Referenced by aaruf_set_erasure_coding_auto().

◆ ec_accumulate_data_block()

void ec_accumulate_data_block	(	aaruformat_context *	ctx,
		const BlockHeader *	block_header,
		const uint8_t *	lzma_props,
		const uint8_t *	payload,
		uint32_t	payload_size,
		uint64_t	file_offset )

Accumulate parity for a data block that was just written to disk.

Called from aaruf_close_current_block() after writing BlockHeader + payload to the file but before freeing the compressed data buffer.

Parameters

ctx	Context with EC enabled.
block_header	Pointer to the BlockHeader that was written.
lzma_props	LZMA properties (may be NULL if not LZMA).
payload	Compressed payload data (or uncompressed if no compression).
payload_size	Size of the payload data.
file_offset	File offset where the block was written.

Definition at line 209 of file erasure.c.

Referenced by aaruf_close_current_block(), and ec_accumulate_data_block().

◆ ec_collect_blocks_by_type()

void ec_collect_blocks_by_type	(	aaruformat_context *	ctx,
		uint32_t	block_type,
		uint64_t **	out_offsets,
		uint32_t **	out_sizes,
		uint32_t *	out_count )

static

Collect file offsets and on-disk sizes for index entries matching a block type.

Definition at line 537 of file erasure.c.

References aaruf_fseek(), IndexEntry::blockType, BlockHeader::cmpLength, DdtHeader2::cmpLength, DeDuplicationTable2, DeDuplicationTableSecondary, ec_collect_blocks_by_type(), aaruformat_context::imageStream, aaruformat_context::index_entries, and IndexEntry::offset.

Referenced by ec_collect_blocks_by_type(), and ec_finalize().

◆ ec_finalize()

void ec_finalize ( aaruformat_context * ctx )

Flush all partial data stripes and write parity for all groups + ECMB + recovery footer.

Definition at line 585 of file erasure.c.

Referenced by aaruf_finalize_write(), and ec_finalize().

◆ ec_flush_data_stripe()

void ec_flush_data_stripe	(	aaruformat_context *	ctx,
		uint32_t	slot )

Write M parity blocks for a completed data stripe slot and record the stripe descriptor.

Parameters

ctx	Context with EC enabled.
slot	Stripe slot index (0 .. K-1).

Definition at line 323 of file erasure.c.

Referenced by ec_accumulate_data_block(), ec_finalize(), and ec_flush_data_stripe().

◆ ec_free()

void ec_free ( aaruformat_context * ctx )

Free all erasure coding state from the context.

Called from aaruf_close().

Parameters

ctx Context.

Definition at line 934 of file erasure.c.

Referenced by aaruf_close(), and ec_free().

◆ ec_load_ecmb()

void ec_load_ecmb ( aaruformat_context * ctx )

Try to load the ECMB from the recovery footer at EOF.

Reads the last 160 bytes, checks for footerMagic, parses ECMB, and populates the ec_read_* fields and block lookup hashmap. Called from aaruf_open().

Definition at line 1046 of file erasure.c.

Referenced by aaruf_open(), and ec_load_ecmb().

◆ ec_recover_data_block()

int32_t ec_recover_data_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint64_t	offset,
		uint8_t *	data,
		uint32_t *	length,
		uint8_t	sector_status )

Attempt to recover a data block that failed decompression or CRC verification.

Looks up the block's stripe via the ECMB, reads surviving stripe members + parity, RS-decodes the erased shard, and decompresses the recovered block.

Parameters

ctx	Context with ec_recovery_available == true.
block_offset	File offset of the corrupted block.
offset	Sector offset within the block.
data	Output buffer for the recovered sector.
length	Output: bytes written to data.
sector_status	Sector status from DDT.

Returns: AARUF_STATUS_OK on success, negative error code on failure.

Definition at line 1303 of file erasure.c.

Referenced by aaruf_read_sector(), and ec_recover_data_block().

◆ ec_recover_ddt_block()

int32_t ec_recover_ddt_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size )

Attempt to recover a DDT secondary block at a given file offset.

Definition at line 918 of file erasure.c.

References AARUF_ERROR_CANNOT_READ_BLOCK, aaruformat_context::ec_ddt_block_lookup, aaruformat_context::ec_ddt_K, aaruformat_context::ec_ddt_M, aaruformat_context::ec_ddt_shard_size, aaruformat_context::ec_ddt_stripe_count, aaruformat_context::ec_ddt_stripes, ec_recover_ddt_block(), and ec_recover_raw_block().

Referenced by decode_ddt_entry_v2(), and ec_recover_ddt_block().

◆ ec_recover_meta_block()

int32_t ec_recover_meta_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size )

Attempt to recover a metadata/media-tag block at a given file offset.

Convenience wrapper around ec_recover_raw_block using the metadata group.

Definition at line 909 of file erasure.c.

References AARUF_ERROR_CANNOT_READ_BLOCK, aaruformat_context::ec_meta_block_lookup, aaruformat_context::ec_meta_K, aaruformat_context::ec_meta_M, aaruformat_context::ec_meta_shard_size, aaruformat_context::ec_meta_stripe_count, aaruformat_context::ec_meta_stripes, ec_recover_meta_block(), and ec_recover_raw_block().

Referenced by ec_recover_meta_block(), process_data_block(), process_tracks_block(), and recover_dpm_block().

◆ ec_recover_raw_block()

int32_t ec_recover_raw_block	(	aaruformat_context *	ctx,
		uint64_t	block_offset,
		uint8_t **	recovered_data,
		uint32_t *	recovered_size,
		void *	stripes,
		uint32_t	stripe_count,
		void *	lookup,
		uint16_t	group_K,
		uint16_t	group_M,
		uint32_t	group_shard_size )

Attempt to recover a raw block at a given file offset using a specified stripe group.

Reads surviving stripe members + parity, RS-decodes the erased shard, returns the recovered raw on-disk bytes.

Parameters

ctx	Context.
block_offset	File offset of the corrupted block.
recovered_data	Output: malloc'd buffer with recovered bytes (caller frees). NULL on failure.
recovered_size	Output: size of recovered data.
stripes	EcReadStripe array for the group.
stripe_count	Number of stripes.
lookup	Block lookup hashmap for the group.
group_K	K for this group.
group_M	M for this group.
group_shard_size	Shard size for this group.

Returns: 0 on success, negative on failure.

Definition at line 827 of file erasure.c.

References aaruf_crc64_data(), AARUF_ERROR_CANNOT_READ_BLOCK, AARUF_ERROR_NOT_ENOUGH_MEMORY, aaruf_fseek(), AARUF_STATUS_OK, EcReadStripe::actual_k, EcReadStripe::data_entries, ec_recover_raw_block(), aaruformat_context::ec_recovery_available, aaruformat_context::ec_recovery_in_progress, aaruformat_context::imageStream, BlockHeader::length, StripeDataBlockEntry::offset, StripeDataBlockEntry::onDiskSize, EcReadStripe::parity_offsets, EcBlockLookupEntry::position, rs_create(), rs_decode(), rs_free(), StripeDataBlockEntry::shardCrc64, and EcBlockLookupEntry::stripe_index.

Referenced by ec_recover_ddt_block(), ec_recover_meta_block(), and ec_recover_raw_block().

◆ ec_write_batch_parity()

void ec_write_batch_parity	(	aaruformat_context *	ctx,
		const uint64_t *	offsets,
		const uint32_t *	sizes,
		uint32_t	block_count,
		uint8_t	group_type,
		uint16_t	parity_data_type,
		uint8_t **	out_desc,
		size_t *	out_desc_len,
		StripeGroupDescriptor *	out_group )

static

Compute and write batch parity for a set of blocks already on disk.

Reads each block's on-disk bytes from file, computes RS parity, writes M parity DLBKs, returns serialized stripe descriptor bytes.

Definition at line 439 of file erasure.c.

References aaruf_crc64_data(), aaruf_fseek(), aaruf_ftell(), DdtHeader2::blockAlignmentShift, BlockHeader::cmpCrc64, BlockHeader::cmpLength, BlockHeader::compression, BlockHeader::crc64, DataBlock, aaruformat_context::dirty_index_block, aaruformat_context::ec_M, ec_write_batch_parity(), StripeGroupDescriptor::groupType, BlockHeader::identifier, aaruformat_context::imageStream, aaruformat_context::index_entries, StripeGroupDescriptor::interleaveDepth, StripeGroupDescriptor::K, kCompressionNone, BlockHeader::length, StripeGroupDescriptor::M, StripeDataBlockEntry::offset, StripeParityBlockEntry::offset, StripeDataBlockEntry::onDiskSize, rs_create(), rs_encode_incremental(), rs_free(), rs_get_coefficient(), StripeDataBlockEntry::shardCrc64, StripeGroupDescriptor::shardSize, StripeGroupDescriptor::stripeCount, BlockHeader::type, and aaruformat_context::user_data_ddt_header.

Referenced by ec_finalize(), and ec_write_batch_parity().

Variable Documentation

◆ ec_stripe_icd

UT_icd ec_stripe_icd = {sizeof(uint8_t), NULL, NULL, NULL}

static

Definition at line 65 of file erasure.c.

Referenced by aaruf_set_erasure_coding().

Data Structures

Macros

Typedefs

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ WRITE_GROUP

Typedef Documentation

◆ EcBlockLookupEntry

◆ EcReadStripe

Function Documentation

◆ aaruf_set_erasure_coding()

◆ aaruf_set_erasure_coding_auto()

◆ ec_accumulate_data_block()

◆ ec_collect_blocks_by_type()

◆ ec_finalize()

◆ ec_flush_data_stripe()

◆ ec_free()

◆ ec_load_ecmb()

◆ ec_recover_data_block()

◆ ec_recover_ddt_block()

◆ ec_recover_meta_block()

◆ ec_recover_raw_block()

◆ ec_write_batch_parity()

Variable Documentation

◆ ec_stripe_icd