mirror of
https://github.com/aaru-dps/libaaruformat.git
synced 2026-11-11 06:45:10 +00:00
231 lines
9.7 KiB
Plaintext
231 lines
9.7 KiB
Plaintext
=== 🛡️ Erasure Coding Map Block (`ECMB`)
|
||
|
||
This block stores the master recovery map for erasure-coded images.
|
||
It enables reconstruction of corrupted data blocks, DDT tables, metadata blocks, and the index using Reed-Solomon or XOR parity computed over the raw on-disk bytes (block header + compressed payload).
|
||
|
||
The ECMB is *not* referenced by the index, because the index itself may need recovery.
|
||
Instead, it is located via the <<Recovery Footer>> at the end of the file.
|
||
A duplicate copy of the ECMB is written immediately after the primary copy for redundancy.
|
||
|
||
==== Feature Flag
|
||
|
||
Erasure coding uses the `featureCompatibleRo` field in the image header:
|
||
|
||
[cols="2,2,6",options="header"]
|
||
|===
|
||
|Bit |Name |Description
|
||
|0 |`AARU_FEATURE_ROCOMPAT_ERASURE` |Image contains erasure coding parity blocks, ECMB, and recovery footer. Readers that do not understand this bit SHOULD open the image read-only but CAN safely ignore the parity data.
|
||
|===
|
||
|
||
Older readers that do not understand this bit will skip the parity blocks and ECMB, and read the image normally.
|
||
|
||
==== Design Rationale
|
||
|
||
Parity is computed on the *raw on-disk bytes* (block header + compressed payload), not on uncompressed sector data.
|
||
This is because a single bit flip in a compressed LZMA, Zstandard, or FLAC block causes decompression to fail entirely—uncompressed parity would be useless for the exact scenario it is supposed to fix.
|
||
Recovery reconstructs the original on-disk bytes, then decompresses normally.
|
||
|
||
==== Structure Definition
|
||
|
||
[source,c]
|
||
----
|
||
#define ECMB_MAGIC 0x424D4345 // "ECMB" in little-endian ASCII
|
||
|
||
typedef struct ErasureCodingMapHeader
|
||
{
|
||
uint32_t identifier; // Block identifier, must be 0x424D4345
|
||
uint8_t algorithm; // Erasure coding algorithm
|
||
uint8_t stripeGroupCount; // Number of stripe groups in payload
|
||
uint16_t compression; // Compression algorithm for the mapping payload
|
||
uint64_t cmpLength; // Size in bytes of the compressed mapping payload
|
||
uint64_t length; // Size in bytes of the uncompressed mapping payload
|
||
uint64_t cmpCrc64; // CRC64-ECMA of the compressed mapping payload
|
||
uint64_t crc64; // CRC64-ECMA of the uncompressed mapping payload
|
||
} ErasureCodingMapHeader;
|
||
----
|
||
|
||
==== Field Descriptions
|
||
|
||
[cols="2,2,2,6",options="header"]
|
||
|===
|
||
|Type |Size |Name |Description
|
||
|uint32_t |4 bytes |identifier |Block identifier, always `ECMB` (0x424D4345).
|
||
|uint8_t |1 byte |algorithm |Erasure coding algorithm (0=XOR, 1=Reed-Solomon Vandermonde).
|
||
|uint8_t |1 byte |stripeGroupCount |Number of stripe groups in the payload (typically 1–5).
|
||
|uint16_t |2 bytes |compression |Compression algorithm used for the payload (same enumeration as data blocks).
|
||
|uint64_t |8 bytes |cmpLength |Size of the compressed payload in bytes.
|
||
|uint64_t |8 bytes |length |Size of the uncompressed payload in bytes.
|
||
|uint64_t |8 bytes |cmpCrc64 |CRC64-ECMA of the compressed payload.
|
||
|uint64_t |8 bytes |crc64 |CRC64-ECMA of the uncompressed payload.
|
||
|===
|
||
|
||
==== Erasure Coding Algorithms
|
||
|
||
[cols="1,2,6",options="header"]
|
||
|===
|
||
|Value |Name |Description
|
||
|0 |XOR |Simple XOR parity. M must be 1. Fastest but tolerates only a single block loss per stripe.
|
||
|1 |RS-Vandermonde |Reed-Solomon with Vandermonde generator matrix over GF(2^8). Supports M ≥ 1. Tolerates up to M block losses per stripe. See Annex K for details.
|
||
|===
|
||
|
||
==== Payload Structure
|
||
|
||
The payload (after decompression) contains `stripeGroupCount` consecutive stripe groups.
|
||
Each group starts with a `StripeGroupDescriptor`, followed by its stripe descriptors.
|
||
|
||
===== Stripe Group Descriptor
|
||
|
||
[source,c]
|
||
----
|
||
typedef struct StripeGroupDescriptor
|
||
{
|
||
uint8_t groupType; // Protection group type
|
||
uint16_t K; // Number of data blocks per stripe
|
||
uint16_t M; // Number of parity blocks per stripe
|
||
uint32_t shardSize; // Fixed shard size in bytes
|
||
uint32_t stripeCount; // Number of stripes in this group
|
||
uint16_t interleaveDepth; // Interleave depth D
|
||
} StripeGroupDescriptor;
|
||
----
|
||
|
||
[cols="2,2,2,6",options="header"]
|
||
|===
|
||
|Type |Size |Name |Description
|
||
|uint8_t |1 byte |groupType |Protection group type (see table below).
|
||
|uint16_t |2 bytes |K |Number of data blocks per stripe.
|
||
|uint16_t |2 bytes |M |Number of parity blocks per stripe.
|
||
|uint32_t |4 bytes |shardSize |Fixed shard size in bytes (maximum possible on-disk block size for this group). Actual blocks are zero-padded to this size for RS math.
|
||
|uint32_t |4 bytes |stripeCount |Number of stripes in this group.
|
||
|uint16_t |2 bytes |interleaveDepth |Interleave depth D. D=K for full interleave (data blocks), D=1 for consecutive (batch groups).
|
||
|===
|
||
|
||
===== Protection Group Types
|
||
|
||
[cols="1,2,6",options="header"]
|
||
|===
|
||
|Value |Name |Description
|
||
|0 |Data |User data blocks (DBLK with DataType=UserData). Interleaved stripes.
|
||
|1 |DDT-Secondary |Secondary DDT subtables. Batch parity at finalization.
|
||
|2 |DDT-Primary |Primary DDT block. K=1, M replicas.
|
||
|3 |Metadata |Media tags, checksums, tracks, geometry, and other metadata blocks. Single batch stripe.
|
||
|4 |Index |Index block. K=1, M replicas—strongest protection for this critical structure.
|
||
|===
|
||
|
||
===== Stripe Descriptor
|
||
|
||
Each stripe contains `actualK` data block entries followed by `M` parity block entries:
|
||
|
||
[source,c]
|
||
----
|
||
// Per stripe:
|
||
uint16_t actualK; // Actual data blocks in this stripe (≤ K; last stripe may be partial)
|
||
|
||
// Repeated actualK times:
|
||
typedef struct StripeDataBlockEntry
|
||
{
|
||
uint64_t offset; // Absolute file offset of the data block
|
||
uint32_t onDiskSize; // Actual on-disk bytes (header + compressed payload)
|
||
uint64_t shardCrc64; // CRC64-ECMA of on-disk bytes zero-padded to shardSize
|
||
} StripeDataBlockEntry;
|
||
|
||
// Repeated M times:
|
||
typedef struct StripeParityBlockEntry
|
||
{
|
||
uint64_t offset; // Absolute file offset of the parity DBLK
|
||
} StripeParityBlockEntry;
|
||
----
|
||
|
||
The `shardCrc64` is computed over the on-disk bytes (block header + compressed payload), zero-padded to `shardSize`.
|
||
This is independent of `BlockHeader.cmpCrc64`—it can detect corruption even when the `BlockHeader` itself is garbled.
|
||
|
||
==== Parity Block Storage
|
||
|
||
Parity shards are stored as standard data blocks (`DBLK`) with the following `DataType` values:
|
||
|
||
[cols="1,2,6",options="header"]
|
||
|===
|
||
|Value |Name |Description
|
||
|104 |ErasureParity |Parity for user data blocks.
|
||
|105 |ErasureParityDdt |Parity for DDT secondary blocks.
|
||
|106 |ErasureParityDdtPrimary |Parity for DDT primary block.
|
||
|107 |ErasureParityMeta |Parity for metadata blocks.
|
||
|108 |ErasureParityIndex |Parity for the index block.
|
||
|===
|
||
|
||
Parity blocks may be compressed using the same algorithm as data blocks (LZMA or Zstandard).
|
||
They often compress well because they contain structured data and large zero-padded tails.
|
||
|
||
==== Interleaved Stripe Assignment (Data Group)
|
||
|
||
For the data group, blocks are assigned to stripe slots using round-robin:
|
||
|
||
----
|
||
Block i → stripe slot (i mod K)
|
||
----
|
||
|
||
This means blocks in the same stripe are K positions apart in the file.
|
||
A contiguous corruption of C blocks affects at most ⌈C/K⌉+1 blocks in any single stripe.
|
||
With K=16 and M=2, you need 33+ contiguously corrupted blocks before any stripe becomes unrecoverable.
|
||
|
||
==== Recovery Footer
|
||
|
||
The last 160 bytes of the file contain the recovery footer, enabling the ECMB to be found even when the header or index is destroyed:
|
||
|
||
[source,c]
|
||
----
|
||
typedef struct AaruRecoveryFooter
|
||
{
|
||
uint64_t ecmbOffset; // Absolute file offset of the primary ECMB
|
||
uint64_t ecmbLength; // Total on-disk size of the ECMB (header + payload)
|
||
uint64_t headerCrc64; // CRC64-ECMA of the original AaruHeaderV2
|
||
AaruHeaderV2 backupHeader; // Complete copy of AaruHeaderV2 (128 bytes)
|
||
uint64_t footerMagic; // Must be 0x52464D4345525641 ("AVRECMFR")
|
||
} AaruRecoveryFooter;
|
||
----
|
||
|
||
[cols="2,2,2,6",options="header"]
|
||
|===
|
||
|Type |Size |Name |Description
|
||
|uint64_t |8 bytes |ecmbOffset |Absolute file offset of the primary ECMB.
|
||
|uint64_t |8 bytes |ecmbLength |Total on-disk size of the ECMB (header + payload).
|
||
|uint64_t |8 bytes |headerCrc64 |CRC64-ECMA of the original `AaruHeaderV2` at file offset 0.
|
||
|AaruHeaderV2 |128 bytes |backupHeader |Complete backup copy of the file header.
|
||
|uint64_t |8 bytes |footerMagic |Must be `0x52464D4345525641` ("AVRECMFR" in ASCII little-endian).
|
||
|===
|
||
|
||
A duplicate ECMB is written at `ecmbOffset + ecmbLength` (aligned) for additional redundancy.
|
||
|
||
==== Recovery Chain
|
||
|
||
The recovery chain defines the fallback sequence when structures are damaged:
|
||
|
||
[cols="3,7",options="header"]
|
||
|===
|
||
|Scenario |Recovery Path
|
||
|Normal operation |Header → `indexOffset` → Index → ECMB (via footer) → verify/recover
|
||
|Header corrupt |Footer `backupHeader` (verified by `headerCrc64`) → same as normal
|
||
|Index corrupt |Footer `ecmbOffset` → ECMB → index parity stripe → RS-reconstruct index
|
||
|DDT corrupt |ECMB → DDT parity stripe → RS-reconstruct DDT block
|
||
|Data block corrupt |ECMB → data parity stripe → RS-reconstruct on-disk bytes → decompress
|
||
|Metadata corrupt |ECMB → metadata parity stripe → RS-reconstruct
|
||
|ECMB corrupt |Duplicate ECMB at secondary offset
|
||
|===
|
||
|
||
==== File Layout
|
||
|
||
The erasure coding structures are written after the index block, at the end of the file:
|
||
|
||
----
|
||
[AaruHeaderV2] offset 0, 128 bytes
|
||
[Data blocks + interleaved parity] variable size
|
||
[DDT blocks (all levels)] variable size
|
||
[DDT parity blocks] M blocks
|
||
[Metadata blocks] variable size
|
||
[Metadata parity blocks] M blocks
|
||
[Index block] at header.indexOffset
|
||
[Index parity blocks] M blocks
|
||
[ECMB (primary)] at footer.ecmbOffset
|
||
[ECMB (duplicate)] at ecmbOffset + ecmbLength (aligned)
|
||
[Recovery Footer] last 160 bytes of file
|
||
----
|