aaru-dps/libaaruformat
1
0
Fork 0
mirror of https://github.com/aaru-dps/libaaruformat.git synced 2026-02-03 21:23:35 +00:00
Code Issues 3 Packages Projects Releases Wiki Activity

Add flux implementation

Browse Source
This commit is contained in:
Rebecca Wallander
2025-10-18 22:04:15 +02:00
parent 6a15b73cd5
commit 65d34b1e9a
20 changed files with 2446 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -234,7 +234,8 @@ add_library(aaruformat SHARED
src/metadata.c
src/dump.c
include/aaruformat/structs/tape.h
src/blocks/tape.c)
src/blocks/tape.c
src/blocks/flux.c)
# Set up include directories for the target
target_include_directories(aaruformat

77
docs/spec/blocks/datastream_payload.adoc Normal file
View File

@@ -0,0 +1,77 @@
=== Data Stream Payload Block (`DSPL`)
This block contains a generic data stream payload. Can be used for data streams such as bitstreams or flux data.
Each `DataStreamPayloadBlock` stores a compressed or uncompressed stream of binary data of variable length.
The block may be compressed using LZMA compression if compression is enabled for the image.
For flux captures, the payload data consists of:
* Data buffer: Raw flux transition timing data
* Index buffer: Index structure mapping logical positions to offsets within the data buffer
These two buffers are concatenated: `[data_buffer][index_buffer]` before compression and storage.
For other data types, the payload layout is specific to the data type.
==== Structure Definition
[source,c]
#define DATA_STREAM_PAYLOAD_MAGIC 0x4C505344
/** Data stream payload block header */
typedef struct DataStreamPayloadHeader
{
uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344).
uint16_t compression; ///< Compression type (0 = None, 1 = Lzma).
uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compressed).
uint32_t length; ///< Uncompressed length in bytes.
uint64_t cmpCrc64; ///< CRC64-ECMA checksum of the compressed payload (or same as crc64 if uncompressed).
uint64_t crc64; ///< CRC64-ECMA checksum of the uncompressed payload.
} DataStreamPayloadHeader;
==== Field Descriptions
[cols="2,2,2,6",options="header"]
|===
|Type
|Size
|Name
|Description
|uint32_t
|4 bytes
|identifier
|The data stream payload block identifier, always `DSPL` (`0x4C505344`)
|uint16_t
|2 bytes
|compression
|Compression type: 0 = None, 1 = Lzma
|uint32_t
|4 bytes
|cmpLength
|Compressed length in bytes (includes LZMA properties if compression = Lzma)
|uint32_t
|4 bytes
|length
|Uncompressed length in bytes (for flux captures: total of data buffer + index buffer)
|uint64_t
|8 bytes
|cmpCrc64
|CRC64-ECMA checksum of the compressed payload data
|uint64_t
|8 bytes
|crc64
|CRC64-ECMA checksum of the uncompressed payload data (for flux captures: data + index buffers)
|===
==== Payload Data
The payload data immediately follows the `DataStreamPayloadHeader`. If compression is Lzma, the first 5 bytes contain LZMA properties, followed by the compressed data. If compression is None, the payload data is stored uncompressed.
For flux captures, the uncompressed payload consists of:
* Data buffer: `length - indexOffset` bytes of flux transition timing data
* Index buffer: `indexOffset` bytes of index structure data
The `indexOffset` value from the corresponding `FluxEntry` indicates where the index buffer starts within the payload. For other data types, the payload layout is specific to the data type.

67
docs/spec/blocks/flux.adoc
View File

@@ -1,6 +1,6 @@
=== Flux Data Block (`FLUX`)
This block lists all known flux captures.
This block contains metadata for all flux captures in the image.
Certain hardware devices, such as Kryoflux, Pauline, and Applesauce, read magnetic media at the flux transition level.
Flux transition reads are digital representations of the analog properties of the media, and cannot be reliably interpreted on a sector-by-sector basis without further processing.
@@ -14,13 +14,24 @@ Flux data is represented as an array of `uint8_t` bytes.
Each byte stores the tick count since the last flux transition.
If no transition is detected within a byte's range, the value `0xFF` is used, and counting resumes in the next byte with ticks accumulated.
Flux data is stored in `DataBlocks` of the flux data type, referenced from a deduplication table of the same type.
Only one flux-type deduplication table is allowed per image, and it must have exactly one level.
The flux capture system uses two block types:
* `FluxDataBlock` (`FLUX` / `0x58554C46`): Contains metadata entries that describe all flux captures in the image
* `DataStreamPayloadBlock` (`DSPL` / `0x4C505344`): Contains the actual flux data payload (data and index buffers) for individual captures. This is a generic block type that can also be used for other data streams.
Each flux capture has one entry in the `FluxDataBlock` and one corresponding `DataStreamPayloadBlock` containing its data.
The `FluxEntry` structure in the data block contains a `payloadOffset` field that points to the file offset where the corresponding `DataStreamPayloadBlock` is stored.
==== Structure Definition
[source,c]
/* Undefined */
#define FLUX_DATA_MAGIC 0x58554C46
/** Flux data block header */
typedef struct FluxHeader
{
uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46).
uint16_t entries; ///< Number of FluxEntry records following this header.
uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded).
} FluxHeader;
==== Field Descriptions
@@ -34,23 +45,34 @@ Only one flux-type deduplication table is allowed per image, and it must have ex
|uint32_t
|4 bytes
|identifier
|The flux data block identifier, always `FLUX`
|The flux data block identifier, always `FLUX` (`0x58554C46`)
|uint16_t
|2 bytes
|entries
|The number of entries following this header
|The number of flux entry records following this header
|uint64_t
|8 bytes
|crc64
|The CRC64-ECMA checksum of the data following this header
|The CRC64-ECMA checksum of the FluxEntry array following this header
|===
==== Flux entries
[source,c]
/* Undefined */
/** Flux capture entry, describes one flux capture in the image */
typedef struct FluxEntry
{
uint32_t head; ///< Head number the capture corresponds to.
uint16_t track; ///< Track number the capture corresponds to.
uint8_t subtrack; ///< Subtrack number the capture corresponds to.
uint32_t captureIndex; ///< Capture index, allows multiple captures for the same location.
uint64_t indexResolution; ///< Resolution in picoseconds for the index stream.
uint64_t dataResolution; ///< Resolution in picoseconds for the data stream.
uint64_t indexOffset; ///< Byte offset within the payload where the index buffer starts.
uint64_t payloadOffset; ///< File offset where the DataStreamPayloadBlock for this capture is stored.
} FluxEntry;
==== Field Descriptions
@@ -64,25 +86,40 @@ Only one flux-type deduplication table is allowed per image, and it must have ex
|uint32_t
|4 bytes
|head
|Head the data corresponds to.
|Head number the flux capture corresponds to.
|uint16_t
|2 bytes
|track
|Track the data corresponds to.
|Track number the flux capture corresponds to.
|uint8_t
|1 byte
|subtrack
|Substep of a track that the data corresponds to.
|Subtrack number the flux capture corresponds to.
|uint32_t
|4 bytes
|captureIndex
|Capture index, allows multiple captures for the same head/track/subtrack combination.
|uint64_t
|8 bytes
|resolution
|Number of picoseconds at which the sampling was performed.
|indexResolution
|Resolution in picoseconds at which the index stream was sampled.
|uint64_t
|8 bytes
|tableEntry
|Entry number in the deduplication table where the data corresponding to this flux entry is stored
|dataResolution
|Resolution in picoseconds at which the data stream was sampled.
|uint64_t
|8 bytes
|indexOffset
|Byte offset within the payload block where the index buffer starts (equals data_length).
|uint64_t
|8 bytes
|payloadOffset
|File offset where the DataStreamPayloadBlock containing this capture's data is stored.
|===

4
docs/spec/spec.adoc
View File

@@ -118,6 +118,10 @@ include::blocks/flux.adoc[]
<<<
include::blocks/datastream_payload.adoc[]
<<<
include::blocks/bitstream.adoc[]
<<<

2
docs/spec/version_history.adoc
View File

@@ -98,4 +98,6 @@ Clarify deduplication table sector status.
Add encrypted and decrypted sector status.
Add Aaru Metadata JSON.
Rework the flux workflow.
|===

10
include/aaruformat/context.h
View File

@@ -21,6 +21,7 @@
#include "blake3.h"
#include "crc64.h"
#include "structs/flux.h"
#include "hash_map.h"
#include "lru.h"
#include "md5.h"
@@ -30,6 +31,8 @@
#include "structs.h"
#include "utarray.h"
typedef struct FluxCaptureMapEntry FluxCaptureMapEntry;
/** \file aaruformat/context.h
* \brief Central runtime context structures for libaaruformat (image state, caches, checksum buffers).
*
@@ -304,6 +307,12 @@ typedef struct aaruformat_context
TapePartitionHashEntry *tape_partitions; ///< Hash table root for tape partitions
bool is_tape; ///< True if the image is a tape image
/* Flux data structures */
FluxHeader flux_data_header; ///< Flux data header (if present).
FluxEntry *flux_entries; ///< Array of flux entries (flux_data_header.entries elements).
UT_array *flux_captures; ///< Pending flux capture payloads (write path).
FluxCaptureMapEntry *flux_map; ///< Hash map for flux capture lookup by head/track/subtrack/capture index.
/* Dirty flags (controls write behavior in close.c) */
bool dirty_secondary_ddt; ///< True if secondary DDT tables should be written during close
bool dirty_primary_ddt; ///< True if primary DDT table should be written during close
@@ -327,6 +336,7 @@ typedef struct aaruformat_context
bool dirty_dumphw_block; ///< True if dump hardware block should be written during close
bool dirty_cicm_block; ///< True if CICM metadata block should be written during close
bool dirty_json_block; ///< True if JSON metadata block should be written during close
bool dirty_flux_block; ///< True if flux block should be written during close
bool dirty_index_block; ///< True if index block should be written during close
} aaruformat_context;

12
include/aaruformat/decls.h
View File

@@ -21,6 +21,7 @@
#include "aaru.h"
#include "crc64.h"
#include "enums.h"
#include "md5.h"
#include "sha1.h"
#include "sha256.h"
@@ -88,6 +89,17 @@ AARU_EXPORT uint64_t AARU_CALL aaruf_crc64_data(const uint8_t *data, uint32_t
AARU_EXPORT int32_t AARU_CALL aaruf_get_tracks(const void *context, uint8_t *buffer, size_t *length);
AARU_EXPORT int32_t AARU_CALL aaruf_set_tracks(void *context, TrackEntry *tracks, const int count);
AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *buffer, size_t *length);
AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack,
uint32_t capture_index, uint8_t *index_data,
uint32_t *index_length, uint8_t *data_data,
uint32_t *data_length);
AARU_EXPORT int32_t AARU_CALL aaruf_write_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack,
uint32_t capture_index, uint64_t data_resolution,
uint64_t index_resolution, const uint8_t *data,
uint32_t data_length, const uint8_t *index,
uint32_t index_length);
AARU_EXPORT int32_t AARU_CALL aaruf_clear_flux_captures(void *context);
AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, uint64_t sector_address, bool negative, uint8_t *data,
uint32_t *length, uint8_t *sector_status);

8
include/aaruformat/enums.h
View File

@@ -130,7 +130,9 @@ typedef enum
DvdSectorEdc = 85, ///< DVD Error Detection Code (EDC)
DvdSectorEccPi = 86, ///< DVD Error Correction Code (ECC) Parity of Inner Code (PI)
DvdEccBlockPo = 87, ///< DVD Error Correction Code (ECC) Parity of Outer Code (PO)
DvdPfi2ndLayer = 88 ///< DVD Physical Format Information for the second layer
DvdPfi2ndLayer = 88, ///< DVD Physical Format Information for the second layer
FluxData = 89, ///< Flux data.
BitstreamData = 90 ///< Bitstream data.
} DataType;
/**
@@ -158,7 +160,9 @@ typedef enum
DumpHardwareBlock = 0x2A504D44, ///< Block containing an array of hardware used to create the image.
TapeFileBlock = 0x454C4654, ///< Block containing list of files for a tape image.
TapePartitionBlock = 0x54425054, ///< Block containing list of partitions for a tape image.
AaruMetadataJsonBlock = 0x444D534A ///< Block containing JSON version of Aaru Metadata
AaruMetadataJsonBlock = 0x444D534A, ///< Block containing JSON version of Aaru Metadata
FluxDataBlock = 0x58554C46, ///< Block containing flux data metadata.
DataStreamPayloadBlock = 0x4C505344 ///< Block containing compressed data stream payload (e.g., flux data, bitstreams).
} BlockType;
/**

1
include/aaruformat/errors.h
View File

@@ -68,6 +68,7 @@
#define AARUF_ERROR_TAPE_PARTITION_NOT_FOUND (-29) ///< Requested tape partition not present in image.
#define AARUF_ERROR_METADATA_NOT_PRESENT (-30) ///< Requested metadata not present in image.
#define AARUF_ERROR_INVALID_SECTOR_LENGTH (-31) ///< Sector length is too big.
#define AARUF_ERROR_FLUX_DATA_NOT_FOUND (-32) ///< Requested flux data not present in image.
/** @} */
/** \name Non-fatal sector status codes (non-negative)

1
include/aaruformat/structs.h
View File

@@ -39,6 +39,7 @@
#include "structs/optical.h"
#include "structs/options.h"
#include "structs/tape.h"
#include "structs/flux.h"
#endif // LIBAARUFORMAT_STRUCTS_H

289
include/aaruformat/structs/flux.h Normal file
View File

@@ -0,0 +1,289 @@
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2025 Natalia Portillo.
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
/**
* @file flux.h
* @brief Data structures for flux transition capture support in Aaru disk images.
*
* This header defines structures used to represent flux transition data captured
* from magnetic media. Flux transitions are the raw analog signals read from
* magnetic storage devices (such as floppy disks, hard drives, and tape) before
* they are interpreted into digital data.
*
* **Flux Capture Overview:**
* Certain hardware devices, such as Kryoflux, Pauline, and Applesauce, read
* magnetic media at the flux transition level rather than at the sector level.
* This provides a more complete representation of the analog properties of the
* media, allowing for advanced recovery techniques and preservation of media
* characteristics that would be lost in sector-level imaging.
*
* **Block Structure:**
* Flux data is stored in two block types:
* - FluxDataBlock: Contains metadata entries describing all flux captures in
* the image. Each entry includes location information (head, track, subtrack),
* capture index, resolution data, and file offsets to the payload blocks.
* - DataStreamPayloadBlock: Contains the actual flux data and index buffers for a
* single capture. Payload blocks may be compressed using LZMA compression.
*
* **Data Organization:**
* Each flux capture includes two data streams:
* - Data buffer: Raw flux transition timing data, represented as an array of
* uint8_t bytes where each byte stores the tick count since the last flux
* transition. The value 0xFF indicates no transition within the byte range.
* - Index buffer: Index structure mapping logical positions to offsets within
* the data buffer, enabling efficient access to specific regions of the flux
* data.
*
* **Capture Identification:**
* Flux captures are uniquely identified by a tuple of (head, track, subtrack,
* captureIndex). This allows multiple captures for the same physical location,
* which is useful for capturing multiple revolutions of a floppy disk track or
* different read attempts.
*
* **Resolution:**
* Each capture specifies two resolution values:
* - dataResolution: The sampling resolution in picoseconds for the data stream
* - indexResolution: The sampling resolution in picoseconds for the index stream
*
* These resolutions determine the precision of the timing measurements and are
* hardware-dependent.
*
* **Storage Efficiency:**
* Payload blocks support LZMA compression to reduce storage requirements. The
* compression is transparent to the API user - aaruf_read_flux_capture() handles
* decompression automatically.
*
* **Use Cases:**
* - Preservation of magnetic media at the lowest level possible
* - Recovery of data from damaged or degraded media
* - Analysis of media characteristics and timing variations
* - Emulation and accurate reproduction of original media behavior
* - Forensic imaging where complete bit-level accuracy is required
*
* @note All structures in this file use packed alignment (#pragma pack(push, 1))
* to ensure consistent on-disk layout across different compilers and platforms.
*
* @see BlockType for the identifier constants used in FluxHeader and DataStreamPayloadHeader
* @see aaruf_get_flux_captures() to retrieve metadata for all captures
* @see aaruf_read_flux_capture() to read actual flux data
* @see aaruf_write_flux_capture() to add flux captures during write mode
*/
#ifndef LIBAARUFORMAT_FLUX_H
#define LIBAARUFORMAT_FLUX_H
#include <stdint.h>
#pragma pack(push, 1)
/**
* @struct FluxHeader
* @brief Header structure for a FluxDataBlock containing flux capture metadata.
*
* This structure is the header of a FluxDataBlock, which lists all flux captures
* in the image. The block contains this header followed by an array of FluxEntry
* structures, one for each flux capture.
*
* The header includes a CRC64 checksum computed over the FluxEntry array to ensure
* data integrity. The identifier field must match BlockType::FluxDataBlock
* (0x58554C46, "FLUX" in ASCII).
*
* @note Only one FluxDataBlock is allowed per image.
* @note The entries field is limited to UINT16_MAX (65535) captures per image.
*/
typedef struct FluxHeader
{
uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46, "FLUX").
uint16_t entries; ///< Number of FluxEntry records following this header. Maximum value: 65535.
uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded).
} FluxHeader;
/**
* @struct FluxEntry
* @brief Metadata entry describing a single flux capture in the FluxDataBlock.
*
* This structure describes one flux capture, including its location on the media,
* capture index, sampling resolutions, and file offsets to the payload data. Each
* FluxEntry corresponds to one DataStreamPayloadBlock containing the actual flux data.
*
* **Location Identification:**
* The head, track, and subtrack fields identify the physical location on the media
* where the capture was taken. The captureIndex allows multiple captures for the
* same location (e.g., multiple revolutions of a floppy disk track).
*
* **Payload Access:**
* The payloadOffset field points to the file offset where the corresponding
* DataStreamPayloadBlock is stored. The indexOffset field indicates where the index
* buffer starts within the payload (the payload is stored as [data_buffer][index_buffer]
* concatenated).
*
* **Resolution:**
* Both indexResolution and dataResolution are specified in picoseconds, indicating
* the precision of the timing measurements. These values are hardware-dependent and
* determine how accurately the flux transitions are represented.
*
* @note The combination of (head, track, subtrack, captureIndex) uniquely identifies
* a flux capture within an image.
* @note The indexOffset equals the data_length, as the index buffer immediately
* follows the data buffer in the payload.
*/
typedef struct FluxEntry
{
uint32_t head; ///< Head number the flux capture corresponds to. Typically 0 or 1 for double-sided media.
uint16_t track; ///< Track number the flux capture corresponds to. Track numbering is format-dependent.
uint8_t subtrack; ///< Subtrack number, allowing sub-stepping within a track. Used for fine positioning.
uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location (e.g., multiple revolutions).
uint64_t indexResolution; ///< Resolution in picoseconds at which the index stream was sampled.
uint64_t dataResolution; ///< Resolution in picoseconds at which the data stream was sampled.
uint64_t indexOffset; ///< Byte offset within the payload where the index buffer starts (equals data_length).
uint64_t payloadOffset; ///< File offset where the DataStreamPayloadBlock containing this capture's data is stored.
} FluxEntry;
/**
* @struct FluxCaptureMeta
* @brief Metadata structure returned by aaruf_get_flux_captures().
*
* This structure contains the public metadata for a flux capture, excluding
* internal file offsets. It is used when retrieving metadata for all captures
* in an image without needing to access the actual flux data.
*
* The structure contains the same location and resolution information as FluxEntry,
* but omits the indexOffset and payloadOffset fields which are implementation
* details not needed by API users.
*
* @see aaruf_get_flux_captures() to retrieve metadata for all captures
* @see FluxEntry for the complete structure including file offsets
*/
typedef struct FluxCaptureMeta
{
uint32_t head; ///< Head number the flux capture corresponds to.
uint16_t track; ///< Track number the flux capture corresponds to.
uint8_t subtrack; ///< Subtrack number the flux capture corresponds to.
uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location.
uint64_t indexResolution; ///< Resolution in picoseconds at which the index stream was sampled.
uint64_t dataResolution; ///< Resolution in picoseconds at which the data stream was sampled.
} FluxCaptureMeta;
/**
* @struct FluxCaptureMapEntry
* @brief Internal hash table entry for flux capture lookup.
*
* This structure is used internally by the library to provide O(1) lookup of
* flux captures by their identifier tuple. It maps a FluxCaptureKey to an index
* in the flux_entries array.
*
* @note This structure is opaque to API users and is only used internally.
* @internal
*/
typedef struct FluxCaptureMapEntry FluxCaptureMapEntry;
/**
* @struct DataStreamPayloadHeader
* @brief Header structure for a DataStreamPayloadBlock containing data stream payload.
*
* This structure is the header of a DataStreamPayloadBlock, which contains a generic
* compressed data stream payload. The payload data immediately follows this header and
* may be compressed using LZMA compression. Currently used for flux capture data, but
* can be used for other data streams such as bitstreams or PNG data.
*
* **Compression:**
* The compression field indicates whether the payload is compressed:
* - 0 (None): Payload is stored uncompressed
* - 1 (Lzma): Payload is compressed using LZMA, with LZMA properties stored
* in the first 5 bytes of the compressed data
*
* **Checksums:**
* Two CRC64 checksums are stored:
* - cmpCrc64: Checksum of the compressed payload (or same as crc64 if uncompressed)
* - crc64: Checksum of the uncompressed payload
*
* Both checksums are validated when reading the payload to ensure data integrity.
*
* **Payload Layout:**
* The uncompressed payload is an arbitrary stream of binary data. For flux captures,
* this consists of concatenated data and index buffers: [data_buffer][index_buffer],
* with the indexOffset from the corresponding FluxEntry indicating where the index
* buffer starts. For other data types, the layout is specific to the data type.
*
* @note The identifier field must match BlockType::DataStreamPayloadBlock (0x4C505344, "DSPL").
* @note If compression is Lzma, cmpLength includes the 5-byte LZMA properties header.
*/
typedef struct DataStreamPayloadHeader
{
uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344, "DSPL").
uint16_t compression; ///< Compression type: 0 = None, 1 = Lzma.
uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compression = Lzma).
uint32_t length; ///< Uncompressed length in bytes.
uint64_t cmpCrc64; ///< CRC64-ECMA checksum of the compressed payload (or same as crc64 if uncompressed).
uint64_t crc64; ///< CRC64-ECMA checksum of the uncompressed payload.
} DataStreamPayloadHeader;
/**
* @struct FluxCaptureRecord
* @brief Internal structure for storing flux capture data during write mode.
*
* This structure is used internally by the library to store flux capture data
* in memory before it is written to the image. It combines the FluxEntry metadata
* with pointers to the actual data and index buffers.
*
* The structure is stored in a utarray (ctx->flux_captures) during write mode,
* and the buffers are freed automatically when the record is removed from the
* array or when the array is freed.
*
* @note This structure is used internally and is not part of the public API.
* @note The data_buffer and index_buffer are owned by the utarray and are freed
* automatically via the flux_capture_record_dtor() destructor.
* @internal
*/
typedef struct FluxCaptureRecord
{
FluxEntry entry; ///< Flux entry metadata describing this capture.
uint8_t *data_buffer; ///< Pointer to the flux data buffer. Owned by the utarray, freed automatically.
uint32_t data_length; ///< Length of the data buffer in bytes.
uint8_t *index_buffer; ///< Pointer to the flux index buffer. Owned by the utarray, freed automatically.
uint32_t index_length; ///< Length of the index buffer in bytes.
} FluxCaptureRecord;
/**
* @struct FluxCaptureKey
* @brief Key structure for flux capture lookup map.
*
* This structure uniquely identifies a flux capture within an image. It is used
* as the key in the internal hash table (flux_map) that provides O(1) lookup
* of flux captures by their identifier tuple.
*
* The structure matches the first four fields of FluxEntry (head, track, subtrack,
* captureIndex).
*
* @note This structure is used internally for efficient lookup and is not part of
* the public API.
* @note The combination of (head, track, subtrack, captureIndex) must be unique
* within an image.
* @internal
*/
typedef struct FluxCaptureKey
{
uint32_t head; ///< Head number identifying the capture location.
uint16_t track; ///< Track number identifying the capture location.
uint8_t subtrack; ///< Subtrack number identifying the capture location.
uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location.
} FluxCaptureKey;
#pragma pack(pop)
#endif // LIBAARUFORMAT_FLUX_H

2
include/internal.h
View File

@@ -39,6 +39,8 @@ void process_dumphw_block(aaruformat_context *ctx, const IndexEntry *entry)
void process_checksum_block(aaruformat_context *ctx, const IndexEntry *entry);
void process_tape_files_block(aaruformat_context *ctx, const IndexEntry *entry);
void process_tape_partitions_block(aaruformat_context *ctx, const IndexEntry *entry);
void process_flux_data_block(aaruformat_context *ctx, const IndexEntry *entry);
int32_t flux_map_rebuild_from_entries(aaruformat_context *ctx);
int32_t decode_ddt_entry_v1(aaruformat_context *ctx, uint64_t sector_address, uint64_t *offset, uint64_t *block_offset,
uint8_t *sector_status);
int32_t decode_ddt_entry_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset,

1111
src/blocks/flux.c Normal file
View File

File diff suppressed because it is too large Load Diff

674
src/close.c
View File

@@ -4185,6 +4185,674 @@ static void write_aaru_json_block(aaruformat_context *ctx)
}
}
/**
* @brief Serialize a single flux capture payload block to the image file.
*
* This helper function writes a DataStreamPayloadBlock containing the raw flux data and index
* buffers for a single flux capture. Flux captures represent analog signal transitions
* recorded from floppy disk drives or other magnetic media, enabling bit-level analysis
* and preservation of timing information that cannot be reconstructed from sector data alone.
*
* The function concatenates the data buffer and index buffer into a single raw payload,
* optionally compresses it using LZMA if compression is enabled, calculates CRC64 checksums
* for both raw and compressed data, writes the DataStreamPayloadHeader followed by the payload
* data, and adds an IndexEntry to enable fast location during subsequent reads.
*
* **Block Structure:**
* The serialized block consists of:
* ```
* +----------------------------------+
* | DataStreamPayloadHeader (32 B) | <- identifier, compression, lengths, CRCs
* +----------------------------------+
* | LZMA Properties (5 B) | <- Only if compression == Lzma
* +----------------------------------+
* | Compressed/Uncompressed | <- Flux data + index buffers
* | Payload Data (variable) |
* +----------------------------------+
* ```
*
* **Processing Flow:**
* 1. **Buffer Concatenation:** Combine data_buffer and index_buffer into a single raw_buffer
* 2. **CRC64 Calculation:** Compute CRC64-ECMA over the raw concatenated buffer
* 3. **Compression Attempt:** If compression enabled, attempt LZMA compression
* - If compression is effective (reduces size), use compressed data
* - If compression is ineffective or fails, fall back to uncompressed
* 4. **Alignment:** Seek to EOF and align to block boundary (blockAlignmentShift)
* 5. **Header Construction:** Build DataStreamPayloadHeader with compression type, lengths, CRCs
* 6. **Write Operations:** Write header, LZMA properties (if compressed), then payload data
* 7. **Indexing:** Add IndexEntry with blockType=DataStreamPayloadBlock, dataType=FluxData
* 8. **Entry Population:** Populate the output FluxEntry with metadata and payload offset
* 9. **Cleanup:** Free temporary compression buffers
*
* **Data and Index Buffers:**
* - data_buffer: Contains the raw flux transition timing data (typically in nanoseconds or
* arbitrary time units based on dataResolution)
* - index_buffer: Contains an index structure that maps logical positions to offsets within
* the data buffer, enabling efficient random access to specific flux transitions
* - The two buffers are concatenated: [data_buffer][index_buffer] before compression/writing
* - indexOffset in the FluxEntry records where the index starts (equals data_length)
*
* **Compression Strategy:**
* - Compression is attempted only if ctx->compression_enabled is true
* - LZMA compression with preset level 9 is used
* - If compressed size >= raw size, compression is disabled and raw data is written
* - LZMA properties (5 bytes) are prepended to compressed data
* - Both raw and compressed data have separate CRC64 checksums for integrity verification
*
* **Alignment Strategy:**
* Before writing, the file position is:
* 1. Moved to EOF using fseek(SEEK_END)
* 2. Aligned forward to next boundary: (position + alignment_mask) & ~alignment_mask
* 3. Where alignment_mask = (1 << blockAlignmentShift) - 1
* This ensures the flux payload block starts on a properly aligned offset for efficient
* I/O and compliance with the Aaru format specification.
*
* **CRC64 Integrity Protection:**
* Two CRC64-ECMA checksums are computed and stored:
* - crc64: Checksum over the raw concatenated buffer (data + index)
* - cmpCrc64: Checksum over the compressed data (or same as crc64 if uncompressed)
* These checksums allow verification of flux payload integrity when reading the image,
* detecting corruption in either the raw or compressed representation.
*
* **Index Entry:**
* On successful write, an IndexEntry is appended to ctx->indexEntries with:
* - blockType = DataStreamPayloadBlock (identifies this as data stream payload)
* - dataType = FluxData (identifies the payload content type)
* - offset = file position where DataStreamPayloadHeader was written
*
* This index entry enables aaruf_read_flux_capture() to quickly locate the payload
* during subsequent reads without scanning the entire file.
*
* **FluxEntry Population:**
* The output @p entry parameter is populated with:
* - head, track, subtrack, captureIndex: Identifiers from the record
* - dataResolution, indexResolution: Timing resolution metadata
* - indexOffset: Offset within payload where index buffer starts (equals data_length)
* - payloadOffset: File offset where this payload block was written
*
* This metadata is later written to the FluxDataBlock to enable efficient lookup
* of flux captures by their identifiers.
*
* **Error Handling:**
* The function returns error codes on failure:
* - AARUF_ERROR_INCORRECT_DATA_SIZE: Raw length exceeds 32-bit limit
* - AARUF_ERROR_NOT_ENOUGH_MEMORY: Memory allocation failure
* - AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER: Failed writing header
* - AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA: Failed writing payload data
*
* On error, all allocated buffers are freed before returning. The function does not
* modify ctx state on failure, allowing the caller to retry or handle the error gracefully.
*
* **Memory Management:**
* - Allocates raw_buffer to hold concatenated data + index buffers
* - Allocates temporary cmp_stream buffer for LZMA compression attempt
* - Allocates compressed_buffer if compression is effective
* - All buffers are freed before function returns, even on error
* - Source buffers in @p record are not modified or freed (managed by caller)
*
* **Thread Safety:**
* This function is NOT thread-safe. It modifies shared ctx state (imageStream file
* position, indexEntries array) and must only be called during single-threaded
* finalization (within write_flux_blocks).
*
* **Use Cases:**
* - Preserving analog flux transitions from floppy disk drives
* - Enabling bit-level analysis and timing reconstruction
* - Supporting forensic analysis of magnetic media
* - Preserving drive-specific timing characteristics
* - Enabling advanced data recovery techniques
*
* **Relationship to Other Functions:**
* - Called by write_flux_blocks() for each flux capture in ctx->flux_captures
* - FluxEntry populated here is later written to FluxDataBlock by write_flux_blocks()
* - aaruf_read_flux_capture() reads these payload blocks during image reading
* - aaruf_write_flux_capture() adds records to ctx->flux_captures during image creation
*
* @param ctx Pointer to an initialized aaruformatContext in write mode. Must not be NULL.
* ctx->imageStream must be open and writable. ctx->indexEntries must be
* initialized (utarray) to accept new index entries.
* @param record Pointer to a FluxCaptureRecord containing the data and index buffers to
* serialize. The record's entry field will be updated with the final
* FluxEntry metadata including payloadOffset.
* @param entry Output parameter that will be populated with the complete FluxEntry
* metadata including the payloadOffset where this block was written.
*
* @return Returns one of the following status codes:
* @retval AARUF_STATUS_OK (0) Successfully wrote the flux payload block. The @p entry
* parameter has been populated with complete metadata.
* @retval AARUF_ERROR_INCORRECT_DATA_SIZE (-5) The combined length of data and index
* buffers exceeds UINT32_MAX (4GB). This should not occur in practice but is
* checked for safety.
* @retval AARUF_ERROR_NOT_ENOUGH_MEMORY (-9) Memory allocation failed for raw buffer,
* compression buffer, or compressed output buffer.
* @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER (-22) Failed to write the
* DataStreamPayloadHeader to the image stream.
* @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA (-23) Failed to write the payload data
* to the image stream.
*
* @note The function does not validate that data_length and index_length match the
* actual buffer sizes. The caller is responsible for ensuring these values are
* correct.
*
* @note Compression effectiveness is evaluated by comparing compressed size to raw size.
* If compression doesn't reduce size, the raw data is written instead. This
* prevents wasting space on incompressible flux data.
*
* @note The payloadOffset stored in the FluxEntry enables direct seeking to the payload
* during reads without requiring a full index scan. This is critical for efficient
* random access to flux captures.
*
* @note LZMA properties are written immediately after the header when compression is
* enabled. The properties are included in cmpLength but not in the separate
* LZMA_PROPERTIES_LENGTH field, so readers must account for this when decompressing.
*
* @warning The function assumes data_buffer and index_buffer are valid for the specified
* lengths. Buffer overruns may occur if the lengths are incorrect. The caller
* must ensure these buffers are properly sized and valid.
*
* @warning Memory allocation failures result in immediate return with an error code.
* No partial writes occur on allocation failure, ensuring image consistency.
*
* @warning Write failures (fwrite returning != 1) result in error codes being returned.
* The caller (write_flux_blocks) is responsible for handling these errors and
* cleaning up any partially written state.
*
* @see DataStreamPayloadHeader for the block header structure definition
* @see FluxEntry for the metadata entry structure
* @see FluxCaptureRecord for the input record structure
* @see write_flux_blocks() for the caller that orchestrates writing all flux captures
* @see aaruf_read_flux_capture() for reading these payload blocks
*
* @internal
*/
static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRecord *record, FluxEntry *entry)
{
uint64_t data_length = record->data_length;
uint64_t index_length = record->index_length;
uint64_t raw_length = data_length + index_length;
if(raw_length > UINT32_MAX)
{
FATAL("Flux capture raw length exceeds 32-bit limit (%" PRIu64 ")", raw_length);
return AARUF_ERROR_INCORRECT_DATA_SIZE;
}
uint8_t *raw_buffer = NULL;
if(raw_length != 0)
{
raw_buffer = malloc((size_t)raw_length);
if(raw_buffer == NULL)
{
FATAL("Could not allocate %" PRIu64 " bytes for flux serialization", raw_length);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
if(data_length != 0 && record->data_buffer != NULL) memcpy(raw_buffer, record->data_buffer, data_length);
if(index_length != 0 && record->index_buffer != NULL)
memcpy(raw_buffer + data_length, record->index_buffer, index_length);
}
uint64_t raw_crc = raw_length != 0 && raw_buffer != NULL ? aaruf_crc64_data(raw_buffer, (size_t)raw_length) : 0;
CompressionType compression = ctx->compression_enabled ? Lzma : None;
uint8_t *compressed_buffer = NULL;
uint32_t cmp_length = 0;
uint64_t cmp_crc = 0;
if(compression == Lzma)
{
size_t cmp_capacity = raw_length ? (size_t)raw_length * 2 + 65536 : LZMA_PROPERTIES_LENGTH + 16;
if(cmp_capacity < raw_length + LZMA_PROPERTIES_LENGTH) cmp_capacity = raw_length + LZMA_PROPERTIES_LENGTH;
uint8_t *cmp_stream = malloc(cmp_capacity);
if(cmp_stream == NULL)
{
free(raw_buffer);
FATAL("Could not allocate %zu bytes for LZMA flux compression", cmp_capacity);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
size_t dst_size = cmp_capacity;
uint8_t lzma_props[LZMA_PROPERTIES_LENGTH] = {0};
size_t props_size = LZMA_PROPERTIES_LENGTH;
int32_t error_no = aaruf_lzma_encode_buffer(cmp_stream, &dst_size, raw_buffer, (size_t)raw_length, lzma_props,
&props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8);
if(error_no != 0 || props_size != LZMA_PROPERTIES_LENGTH || dst_size >= raw_length)
{
TRACE("Flux capture compression fell back to uncompressed (err=%d, dst=%zu, raw=%" PRIu64 ")", error_no,
dst_size, raw_length);
compression = None;
free(cmp_stream);
}
else
{
cmp_length = (uint32_t)(dst_size + LZMA_PROPERTIES_LENGTH);
compressed_buffer = malloc(cmp_length);
if(compressed_buffer == NULL)
{
free(cmp_stream);
free(raw_buffer);
FATAL("Could not allocate %u bytes for flux compressed payload", cmp_length);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
memcpy(compressed_buffer, lzma_props, LZMA_PROPERTIES_LENGTH);
memcpy(compressed_buffer + LZMA_PROPERTIES_LENGTH, cmp_stream, dst_size);
cmp_crc = aaruf_crc64_data(compressed_buffer, cmp_length);
free(cmp_stream);
free(raw_buffer);
raw_buffer = NULL;
}
}
if(compression == None)
{
cmp_length = (uint32_t)raw_length;
cmp_crc = raw_crc;
compressed_buffer = raw_buffer;
raw_buffer = NULL;
}
// Align stream position to block boundary
fseek(ctx->imageStream, 0, SEEK_END);
long payload_position = ftell(ctx->imageStream);
const uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
if(payload_position & alignment_mask)
{
const uint64_t aligned_position = payload_position + alignment_mask & ~alignment_mask;
fseek(ctx->imageStream, aligned_position, SEEK_SET);
payload_position = aligned_position;
}
DataStreamPayloadHeader payload_header = {0};
payload_header.identifier = DataStreamPayloadBlock;
payload_header.compression = (uint16_t)compression;
payload_header.cmpLength = cmp_length;
payload_header.length = (uint32_t)raw_length;
payload_header.cmpCrc64 = cmp_crc;
payload_header.crc64 = raw_crc;
if(fwrite(&payload_header, sizeof(DataStreamPayloadHeader), 1, ctx->imageStream) != 1)
{
free(compressed_buffer);
free(raw_buffer);
FATAL("Could not write flux payload header");
return AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER;
}
if(cmp_length != 0 && compressed_buffer != NULL)
{
if(fwrite(compressed_buffer, cmp_length, 1, ctx->imageStream) != 1)
{
free(compressed_buffer);
free(raw_buffer);
FATAL("Could not write flux payload data");
return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA;
}
}
IndexEntry payload_entry;
payload_entry.blockType = DataStreamPayloadBlock;
payload_entry.dataType = FluxData;
payload_entry.offset = payload_position;
utarray_push_back(ctx->index_entries, &payload_entry);
entry->head = record->entry.head;
entry->track = record->entry.track;
entry->subtrack = record->entry.subtrack;
entry->captureIndex = record->entry.captureIndex;
entry->dataResolution = record->entry.dataResolution;
entry->indexResolution = record->entry.indexResolution;
entry->indexOffset = record->data_length;
entry->payloadOffset = payload_position;
record->entry = *entry;
free(compressed_buffer);
free(raw_buffer);
return AARUF_STATUS_OK;
}
/**
* @brief Serialize all accumulated flux capture blocks to the image file.
*
* This function writes the complete flux capture data structure to the Aaru image file.
* Flux captures represent analog signal transitions recorded from floppy disk drives or
* other magnetic media, preserving timing information that enables bit-level analysis
* and advanced data recovery techniques that cannot be performed using sector data alone.
*
* The function processes all flux captures that have been enqueued in ctx->flux_captures
* during image creation. For each capture, it writes a DataStreamPayloadBlock containing the
* raw flux data and index buffers. After all payload blocks are written, it writes a
* FluxDataBlock containing metadata entries that enable efficient lookup and access to
* the flux captures. Finally, it rebuilds the in-memory flux capture lookup map for
* fast access during the remainder of the close operation.
*
* The flux capture system supports multiple captures per disk (identified by head,
* track, subtrack, and captureIndex), enabling preservation of multiple read attempts,
* different drive characteristics, or verification passes.
*
* **Block Structure:**
* The serialized flux capture structure consists of:
* ```
* +------------------------------+
* | DataStreamPayloadBlock 0 | <- First capture's data + index
* +------------------------------+
* | DataStreamPayloadBlock 1 | <- Second capture's data + index
* +------------------------------+
* | ... |
* +------------------------------+
* | DataStreamPayloadBlock (n-1) | <- Last capture's data + index
* +------------------------------+
* | FluxDataBlock | <- Metadata block with all entries
* | + FluxHeader (16 B) | identifier, entries count, crc64
* | + FluxEntry 0 (32 B) | head, track, subtrack, offsets, etc.
* | + FluxEntry 1 (32 B) |
* | + ... |
* | + FluxEntry (n-1) (32 B) |
* +------------------------------+
* ```
*
* **Processing Flow:**
* 1. **Validation:** Check context validity and presence of flux captures
* 2. **Entry Count Validation:** Verify capture count doesn't exceed UINT16_MAX
* 3. **Entry Array Allocation:** Allocate temporary array for FluxEntry structures
* 4. **Payload Writing:** For each capture, call write_flux_capture_payload() to write
* the payload block and populate the corresponding FluxEntry
* 5. **Header Construction:** Build FluxHeader with entry count and CRC64 over entries
* 6. **Alignment:** Seek to EOF and align to block boundary (blockAlignmentShift)
* 7. **Metadata Write:** Write FluxHeader followed by FluxEntry array
* 8. **Indexing:** Add IndexEntry for FluxDataBlock to enable fast location during reads
* 9. **Context Update:** Update ctx->flux_entries and ctx->flux_data_header
* 10. **Cleanup:** Free ctx->flux_captures array (no longer needed)
* 11. **Map Rebuild:** Rebuild flux capture lookup map from entries for fast access
*
* **Flux Capture Organization:**
* - Each flux capture is identified by: head, track, subtrack, captureIndex
* - Multiple captures can exist for the same track (e.g., different read attempts)
* - The captureIndex allows distinguishing between multiple captures of the same location
* - Entries are written in the order they appear in ctx->flux_captures (insertion order)
*
* **CRC64 Integrity Protection:**
* A CRC64-ECMA checksum is computed over the complete array of FluxEntry structures
* using aaruf_crc64_data(). This checksum is stored in the FluxHeader and verified
* during image opening by process_flux_data_block() to detect corruption in the flux
* metadata. The checksum covers only the entry data, not the header itself.
*
* **Alignment Strategy:**
* Before writing the FluxDataBlock, the file position is:
* 1. Moved to EOF using fseek(SEEK_END)
* 2. Aligned forward to next boundary: (position + alignment_mask) & ~alignment_mask
* 3. Where alignment_mask = (1 << blockAlignmentShift) - 1
* This ensures the flux data block starts on a properly aligned offset for efficient
* I/O and compliance with the Aaru format specification. Individual payload blocks
* are also aligned by write_flux_capture_payload().
*
* **Write Sequence:**
* The function performs a multi-stage write operation:
* 1. Write all DataStreamPayloadBlock entries (one per capture, via write_flux_capture_payload)
* 2. Write FluxHeader (sizeof(FluxHeader) = 16 bytes)
* 3. Write FluxEntry array (capture_count * sizeof(FluxEntry) bytes)
*
* All writes must succeed for the index entry to be added. If any write fails, the
* function returns an error code and the caller (aaruf_close) handles cleanup.
*
* **Indexing:**
* On successful write of the FluxDataBlock, an IndexEntry is appended to ctx->indexEntries:
* - blockType = FluxDataBlock (identifies this as flux capture metadata)
* - dataType = 0 (flux data blocks have no subtype)
* - offset = file position where FluxHeader was written
*
* Individual DataStreamPayloadBlock entries are indexed by write_flux_capture_payload() with
* blockType=DataStreamPayloadBlock and dataType=FluxData.
*
* This index entry enables process_flux_data_block() to quickly locate the flux metadata
* during subsequent image opens without scanning the entire file.
*
* **Flux Capture Lookup Map:**
* After writing, the function calls flux_map_rebuild_from_entries() to rebuild the
* in-memory hash table that maps (head, track, subtrack, captureIndex) tuples to
* FluxEntry array indices. This map enables O(1) lookup of flux captures by their
* identifiers, which is used by aaruf_read_flux_capture() and other flux access functions.
*
* The map is stored in ctx->flux_capture_map and uses UTHASH for efficient hash table
* operations. The map is cleared and rebuilt from scratch each time this function is
* called, ensuring consistency with the newly written entries.
*
* **Error Handling:**
* The function returns error codes on failure:
* - AARUF_ERROR_NOT_AARUFORMAT: Invalid context or image stream
* - AARUF_ERROR_INCORRECT_DATA_SIZE: Capture count exceeds UINT16_MAX
* - AARUF_ERROR_NOT_ENOUGH_MEMORY: Failed to allocate entry array
* - Error codes propagated from write_flux_capture_payload() on payload write failure
* - Error codes propagated from flux_map_rebuild_from_entries() on map rebuild failure
*
* On error, allocated memory is freed and ctx->flux_entries is restored to its previous
* value (if any), ensuring the context remains in a consistent state.
*
* **Memory Management:**
* - Allocates temporary entries array sized to hold all FluxEntry structures
* - Previous ctx->flux_entries (if any) are preserved and restored on error
* - On success, previous entries are freed and replaced with new entries
* - ctx->flux_captures array is freed after successful write (no longer needed)
* - Individual payload buffers are managed by write_flux_capture_payload()
*
* **No-op Conditions:**
* If ctx->flux_captures is NULL or empty (utarray_len == 0), the function returns
* AARUF_STATUS_OK immediately without writing anything. This allows images without
* flux captures to be created normally.
*
* **Thread Safety:**
* This function is NOT thread-safe. It modifies shared ctx state (imageStream file
* position, indexEntries array, flux_entries, flux_data_header, flux_capture_map) and
* must only be called during single-threaded finalization (within aaruf_close).
*
* **Use Cases:**
* - Preserving analog flux transitions from floppy disk drives for forensic analysis
* - Enabling bit-level timing analysis and reconstruction
* - Supporting advanced data recovery from damaged or degraded media
* - Preserving drive-specific characteristics and read variations
* - Enabling research into magnetic media encoding and decoding
* - Supporting preservation of non-standard or copy-protected disk formats
*
* **Relationship to Other Functions:**
* - Flux captures are added via aaruf_write_flux_capture() during image creation
* - Captures are stored in ctx->flux_captures array until image close
* - This function serializes all captures to disk during aaruf_close()
* - process_flux_data_block() reads and reconstructs the entries during aaruf_open()
* - aaruf_read_flux_capture() uses the lookup map to access specific captures
* - flux_map_rebuild_from_entries() rebuilds the lookup map from entries
*
* **Format Considerations:**
* - The FluxDataBlock must be written after all DataStreamPayloadBlock entries
* - The entry count in FluxHeader must match the number of payload blocks written
* - Each FluxEntry's payloadOffset must point to a valid DataStreamPayloadBlock
* - The CRC64 in FluxHeader enables verification of entry array integrity
* - Multiple captures per track are supported via captureIndex differentiation
*
* @param ctx Pointer to an initialized aaruformatContext in write mode. Must not be NULL.
* ctx->flux_captures should contain the array of flux captures to serialize
* (may be NULL or empty if no flux captures were added). ctx->imageStream
* must be open and writable. ctx->indexEntries must be initialized (utarray)
* to accept new index entries.
*
* @return Returns one of the following status codes:
* @retval AARUF_STATUS_OK (0) Successfully wrote all flux capture blocks. This occurs when:
* - The context is valid and imageStream is open
* - All payload blocks were written successfully
* - The FluxDataBlock header and entries were written successfully
* - The flux capture lookup map was rebuilt successfully
* - Or when no flux captures were present (no-op success)
*
* @retval AARUF_ERROR_NOT_AARUFORMAT (-1) The context is invalid or imageStream is NULL.
* This indicates a programming error or corrupted context state.
*
* @retval AARUF_ERROR_INCORRECT_DATA_SIZE (-5) The number of flux captures exceeds
* UINT16_MAX (65535). This is a practical limit to keep the header size reasonable.
*
* @retval AARUF_ERROR_NOT_ENOUGH_MEMORY (-9) Memory allocation failed for the FluxEntry
* array. This can occur when there are extremely many flux captures or system
* memory is exhausted.
*
* @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER (-22) Failed to write the FluxHeader.
* This can occur due to disk full, I/O errors, or stream corruption.
*
* @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA (-23) Failed to write the FluxEntry array.
* This can occur due to disk full, I/O errors, or stream corruption.
*
* @retval <other error codes> Propagated from write_flux_capture_payload() if any
* payload block write fails, or from flux_map_rebuild_from_entries() if map
* rebuild fails.
*
* @note The function preserves ctx->flux_entries if it already exists, freeing the
* previous array only after successful write of the new entries. This ensures
* that read operations can continue to use the old entries if the write fails.
*
* @note The flux capture lookup map is rebuilt even if it already exists, ensuring
* consistency with the newly written entries. The previous map is cleared
* by flux_map_rebuild_from_entries() before rebuilding.
*
* @note Entry ordering in the FluxDataBlock matches the order in ctx->flux_captures,
* which is typically insertion order. Applications reading flux captures should
* not rely on any specific ordering beyond what is guaranteed by the identifiers.
*
* @note The function does not validate that payloadOffset values in FluxEntry structures
* point to valid DataStreamPayloadBlock locations. This validation is performed during
* reading by process_flux_data_block() and aaruf_read_flux_capture().
*
* @warning If write_flux_capture_payload() fails for any capture, this function
* immediately returns the error code without attempting to write remaining
* captures. Partial flux capture data may be present in the image, but the
* FluxDataBlock will not be written, making the flux data inaccessible.
*
* @warning Memory allocation failure for the entries array results in immediate return
* with an error code. No flux data is written in this case, even if some
* payload blocks were already written (though this should not occur as payload
* writing happens before entry array allocation).
*
* @warning The function modifies ctx->flux_entries, ctx->flux_data_header, and
* ctx->flux_capture_map. These modifications occur even if a later step fails,
* so the caller must handle cleanup if needed. However, the function does
* restore ctx->flux_entries to its previous value on error.
*
* @see FluxHeader for the metadata block header structure definition
* @see FluxEntry for individual flux capture entry structure definition
* @see DataStreamPayloadHeader for payload block header structure definition
* @see write_flux_capture_payload() for writing individual payload blocks
* @see flux_map_rebuild_from_entries() for rebuilding the lookup map
* @see process_flux_data_block() for reading flux metadata during image opening
* @see aaruf_write_flux_capture() for adding flux captures during image creation
* @see aaruf_read_flux_capture() for reading flux captures from opened images
*
* @internal
*/
static int32_t write_flux_blocks(aaruformat_context *ctx)
{
TRACE("Entering write_flux_blocks(%p)", ctx);
if(ctx == NULL || ctx->imageStream == NULL)
{
FATAL("Invalid context when writing flux blocks");
return AARUF_ERROR_NOT_AARUFORMAT;
}
if(ctx->flux_captures == NULL || utarray_len(ctx->flux_captures) == 0)
{
TRACE("No flux captures enqueued, skipping flux block serialization");
return AARUF_STATUS_OK;
}
size_t capture_count = utarray_len(ctx->flux_captures);
if(capture_count > UINT16_MAX)
{
FATAL("Flux capture count exceeds header capacity (%zu > %u)", capture_count, UINT16_MAX);
return AARUF_ERROR_INCORRECT_DATA_SIZE;
}
FluxEntry *previous_entries = ctx->flux_entries;
ctx->flux_entries = NULL;
FluxEntry *entries = malloc(capture_count * sizeof(FluxEntry));
if(entries == NULL)
{
ctx->flux_entries = previous_entries;
FATAL("Could not allocate %zu bytes for flux entries", capture_count * sizeof(FluxEntry));
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
size_t idx = 0;
for(FluxCaptureRecord *record = (FluxCaptureRecord *)utarray_front(ctx->flux_captures); record != NULL;
record = (FluxCaptureRecord *)utarray_next(ctx->flux_captures, record), ++idx)
{
int32_t res = write_flux_capture_payload(ctx, record, &entries[idx]);
if(res != AARUF_STATUS_OK)
{
free(entries);
ctx->flux_entries = previous_entries;
return res;
}
}
FluxHeader header = {0};
header.identifier = FluxDataBlock;
header.entries = (uint16_t)capture_count;
header.crc64 =
capture_count == 0 ? 0 : aaruf_crc64_data((const uint8_t *)entries, capture_count * sizeof(FluxEntry));
// Align stream position to block boundary
fseek(ctx->imageStream, 0, SEEK_END);
long metadata_position = ftell(ctx->imageStream);
const uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1;
if(metadata_position & alignment_mask)
{
const uint64_t aligned_position = metadata_position + alignment_mask & ~alignment_mask;
fseek(ctx->imageStream, aligned_position, SEEK_SET);
metadata_position = aligned_position;
}
if(fwrite(&header, sizeof(FluxHeader), 1, ctx->imageStream) != 1)
{
free(entries);
ctx->flux_entries = previous_entries;
FATAL("Could not write flux metadata header");
return AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER;
}
if(capture_count != 0)
{
size_t written_entries = fwrite(entries, sizeof(FluxEntry), capture_count, ctx->imageStream);
if(written_entries != capture_count)
{
free(entries);
ctx->flux_entries = previous_entries;
FATAL("Could not write %zu flux entries (wrote %zu)", capture_count, written_entries);
return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA;
}
}
IndexEntry metadata_entry;
metadata_entry.blockType = FluxDataBlock;
metadata_entry.dataType = 0;
metadata_entry.offset = metadata_position;
utarray_push_back(ctx->index_entries, &metadata_entry);
if(previous_entries != NULL) free(previous_entries);
ctx->flux_entries = entries;
ctx->flux_data_header = header;
utarray_free(ctx->flux_captures);
ctx->flux_captures = NULL;
int32_t map_result = flux_map_rebuild_from_entries(ctx);
if(map_result != AARUF_STATUS_OK) return map_result;
TRACE("Wrote %zu flux captures", capture_count);
return AARUF_STATUS_OK;
}
/**
* @brief Serialize the accumulated index entries at the end of the image and back-patch the header.
*
@@ -4427,6 +5095,12 @@ AARU_EXPORT int AARU_CALL aaruf_close(void *context)
// Write tracks block
if(ctx->dirty_tracks_block) write_tracks_block(ctx);
// Write flux capture blocks
if(ctx->dirty_flux_block) {
res = write_flux_blocks(ctx);
if(res != AARUF_STATUS_OK) return res;
}
// Write MODE 2 subheader data block
if(ctx->dirty_mode2_subheaders_block) write_mode2_subheaders_block(ctx);

1
src/create.c
View File

@@ -587,6 +587,7 @@ AARU_EXPORT void AARU_CALL *aaruf_create(const char *filepath, const uint32_t me
ctx->dirty_dumphw_block = true;
ctx->dirty_cicm_block = true;
ctx->dirty_json_block = true;
ctx->dirty_flux_block = true;
ctx->dirty_index_block = true;
TRACE("Exiting aaruf_create() = %p", ctx);

4
src/open.c
View File

@@ -545,6 +545,10 @@ AARU_EXPORT void AARU_CALL *aaruf_open(const char *filepath, const bool resume_m
case TapePartitionBlock:
process_tape_partitions_block(ctx, entry);
break;
case FluxDataBlock:
process_flux_data_block(ctx, entry);
break;
default:
TRACE("Unhandled block type %4.4s with data type %d is indexed to be at %" PRIu64 "",

4
templates/aaruformat.bt
View File

@@ -701,7 +701,9 @@ enum <ushort> DataType
DvdSectorEdc = 85,
DvdSectorEccPi = 86,
DvdEccBlockPo = 87,
DvdPfi2ndLayer = 88
DvdPfi2ndLayer = 88,
FluxData = 89,
BitstreamData = 90
};
enum <uint> BlockType

53
templates/aaruformat.hexpat
View File

@@ -727,8 +727,13 @@ enum DataType : u16
DecryptedDVDDiscKey = 80,
DVD_CPI_MAI = 81,
DecryptedDVDTitleKey = 82,
FluxData = 83,
BitstreamData = 84
DvdSectorId = 83,
DvdSectorIed = 84,
DvdSectorEdc = 85,
DvdSectorEccPi = 86,
DvdEccBlockPo = 87,
FluxData = 88,
BitstreamData = 89
};
enum BlockType : u32
@@ -753,6 +758,7 @@ enum BlockType : u32
TwinSectorTable = 0x42545754,
CompactDiscIndexesBlock = 0x58494443,
FluxDataBlock = 0x58554C46,
DataStreamPayloadBlock = 0x4C505344,
BitstreamDataBlock = 0x53544942,
TrackLayoutBlock = 0x594C4B54,
JsonMetadataBlock = 0x444D534A
@@ -1130,6 +1136,47 @@ struct CompactDiscIndexesBlock
CompactDiscIndexEntry indexes[entries];
};
struct FluxEntry
{
u32 head;
u16 track;
u8 subtrack;
u32 captureIndex;
u64 indexResolution;
u64 dataResolution;
u64 indexOffset;
u64 payloadOffset;
};
struct FluxHeader
{
BlockType identifier;
u16 entries;
u64 crc64;
};
struct FluxDataBlock
{
FluxHeader header;
FluxEntry entries[header.entries];
};
struct DataStreamPayloadHeader
{
BlockType identifier;
u16 compression;
u32 cmpLength;
u32 length;
u64 cmpCrc64;
u64 crc64;
};
struct DataStreamPayloadBlock
{
DataStreamPayloadHeader header;
u8 payload[header.cmpLength];
};
using Index;
struct IndexEntry
@@ -1153,6 +1200,8 @@ struct IndexEntry
(BlockType::TapePartitionBlock): TapePartitionHeader *tapePartitions : u64 [[inline]];
(BlockType::CompactDiscIndexesBlock): CompactDiscIndexesBlock *compactDiscIndexes : u64 [[inline]];
(BlockType::Index3): Index *index : u64 [[inline]];
(BlockType::FluxDataBlock): FluxDataBlock *fluxData : u64 [[inline]];
(BlockType::DataStreamPayloadBlock): DataStreamPayloadBlock *dataStreamPayload : u64 [[inline]];
(_): BlockType *offset : u64 [[inline]];
}
};

140
tool/convert.c
View File

@@ -1286,6 +1286,146 @@ int convert(const char *input_path, const char *output_path, bool use_long)
print_info("Time elapsed:", elapsed_str);
snprintf(buffer, sizeof(buffer), "%s/sec", speed_str);
print_info("Average speed:", buffer);
// Copy flux captures from input to output
size_t flux_captures_length = 0;
res = aaruf_get_flux_captures(input_ctx, NULL, &flux_captures_length);
if(res == AARUF_ERROR_BUFFER_TOO_SMALL)
{
uint8_t *flux_captures = malloc(flux_captures_length);
if(flux_captures == NULL)
{
printf("\nError allocating memory for flux captures buffer.\n");
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
res = aaruf_get_flux_captures(input_ctx, flux_captures, &flux_captures_length);
if(res == AARUF_STATUS_OK)
{
size_t capture_count = flux_captures_length / sizeof(FluxCaptureMeta);
FluxCaptureMeta *captures = (FluxCaptureMeta *)flux_captures;
uint8_t *index_data = NULL;
uint8_t *data_data = NULL;
uint32_t index_length = 0;
uint32_t data_length = 0;
printf("Copying %zu flux captures...\n", capture_count);
for(size_t i = 0; i < capture_count; i++)
{
// First call to get required buffer sizes
res = aaruf_read_flux_capture(input_ctx, captures[i].head, captures[i].track, captures[i].subtrack,
captures[i].captureIndex, NULL, &index_length, NULL, &data_length);
if(res == AARUF_ERROR_BUFFER_TOO_SMALL)
{
// Allocate buffers
if(index_length > 0)
{
index_data = malloc(index_length);
if(index_data == NULL)
{
printf("Error allocating memory for flux index buffer.\n");
free(flux_captures);
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
}
if(data_length > 0)
{
data_data = malloc(data_length);
if(data_data == NULL)
{
printf("Error allocating memory for flux data buffer.\n");
free(index_data);
free(flux_captures);
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
}
// Read flux capture data
res = aaruf_read_flux_capture(input_ctx, captures[i].head, captures[i].track, captures[i].subtrack,
captures[i].captureIndex, index_data, &index_length, data_data,
&data_length);
if(res == AARUF_STATUS_OK)
{
// Write flux capture to output
res = aaruf_write_flux_capture(output_ctx, captures[i].head, captures[i].track,
captures[i].subtrack, captures[i].captureIndex,
captures[i].dataResolution, captures[i].indexResolution, data_data,
data_length, index_data, index_length);
if(res != AARUF_STATUS_OK)
{
printf("Error %d when writing flux capture (head=%u, track=%u, subtrack=%u, index=%u) to output image.\n",
res, captures[i].head, captures[i].track, captures[i].subtrack,
captures[i].captureIndex);
free(index_data);
free(data_data);
free(flux_captures);
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return res;
}
}
else
{
printf("Error %d when reading flux capture (head=%u, track=%u, subtrack=%u, index=%u) from input image.\n",
res, captures[i].head, captures[i].track, captures[i].subtrack, captures[i].captureIndex);
free(index_data);
free(data_data);
free(flux_captures);
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return res;
}
free(index_data);
free(data_data);
index_data = NULL;
data_data = NULL;
index_length = 0;
data_length = 0;
}
else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND)
{
printf("Error %d when reading flux capture (head=%u, track=%u, subtrack=%u, index=%u) from input image.\n",
res, captures[i].head, captures[i].track, captures[i].subtrack, captures[i].captureIndex);
free(flux_captures);
free(sector_data);
aaruf_close(input_ctx);
aaruf_close(output_ctx);
return res;
}
}
printf("Successfully copied %zu flux captures.\n", capture_count);
}
else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND)
{
printf("\nError %d when getting flux captures from input image.\n", res);
}
free(flux_captures);
}
else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND)
{
printf("\nError %d when getting flux captures from input image.\n", res);
}
printf("\n");
// Clean up
free(sector_data);

4
tool/info.c
View File

@@ -855,6 +855,10 @@ int info(const char *path)
draw_box_bottom(COLOR_HEADER);
}
if(ctx->flux_data_header.entries > 0) {
printf("Image contains %d flux captures.\n", ctx->flux_data_header.entries);
}
// Checksums Section
bool hasChecksums =
ctx->checksums.hasMd5 || ctx->checksums.hasSha1 || ctx->checksums.hasSha256 || ctx->checksums.hasSpamSum;