aaru-dps/libaaruformat
1
0
Fork 0
mirror of https://github.com/aaru-dps/libaaruformat.git synced 2025-12-16 19:24:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement support for index v2.

Browse Source
This commit is contained in:
Natalia Portillo
2025-08-01 21:18:48 +01:00
parent 4f25209145
commit 9a4c04b2ca
8 changed files with 224 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
CMakeLists.txt
View File

@@ -21,24 +21,28 @@ IF (APPLE)
SET(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "Build architectures for Mac OS X" FORCE)
ENDIF ()
ENDIF (APPLE)
IF(APPLE)
IF("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "")
ENDIF()
ENDIF(APPLE)
project(libaaruformat C)
add_compile_definitions(__STDC_FORMAT_MACROS=1)
if ("${CMAKE_C_COMPILER_ID}" MATCHES "MSVC" AND "${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARMV7")
if("${CMAKE_C_COMPILER_ID}" MATCHES "MSVC" AND "${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARMV7")
set(CMAKE_C_STANDARD 11)
else ()
else()
set(CMAKE_C_STANDARD 99)
endif ()
endif()
if ("${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
if ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "arm")
if("${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "arm")
set(WIN32 TRUE)
endif ()
endif()
add_link_options(-static-libgcc)
endif ()
endif()
message("Detected system processor: ${CMAKE_SYSTEM_PROCESSOR}")
message("Detected vs platform name: ${CMAKE_C_COMPILER_ARCHITECTURE_ID}")
@@ -48,65 +52,68 @@ message("Detected platform: ${CMAKE_C_PLATFORM_ID}")
message("Size of (void*): ${CMAKE_SIZEOF_VOID_P}")
# Check if target is 64-bit
if ("${CMAKE_SIZEOF_VOID_P}" MATCHES "8" OR "${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "x64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "AMD64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64")
if("${CMAKE_SIZEOF_VOID_P}" MATCHES "8" OR "${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "x64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "AMD64" OR "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64")
set(ARCHITECTURE_IS_64BIT TRUE)
endif ()
endif()
if ("${CMAKE_BUILD_TYPE}" MATCHES "Release")
if("${CMAKE_BUILD_TYPE}" MATCHES "Release")
add_compile_definitions(NDEBUG)
if ("${CMAKE_C_COMPILER_ID}" MATCHES "MSVC")
if("${CMAKE_C_COMPILER_ID}" MATCHES "MSVC")
add_compile_options("/O2" "/fp:fast")
if (${CMAKE_C_COMPILER_ARCHITECTURE_ID} MATCHES "X86" OR ${CMAKE_C_COMPILER_ARCHITECTURE_ID} MATCHES "x64")
if(${CMAKE_C_COMPILER_ARCHITECTURE_ID} MATCHES "X86" OR ${CMAKE_C_COMPILER_ARCHITECTURE_ID} MATCHES "x64")
add_compile_options("/arch:SSE2")
endif ()
else ()
endif()
else()
add_compile_options(-ffast-math -O3)
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "i686" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "AMD64")
if (NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "i686" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "AMD64")
if(NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
add_compile_options(-march=core2 -mtune=westmere -mfpmath=sse)
endif ()
endif()
add_compile_options(-msse3)
if (NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
if(NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
add_compile_options(-flto)
endif ()
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
if (NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
if(NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
add_compile_options(-march=armv8-a)
endif ()
endif()
if (NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
if(NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
add_compile_options(-flto)
endif ()
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7l" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
if (NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7l" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
if(NOT "${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")
add_compile_options(-march=armv7+fp -mfpu=vfpv3-d16)
endif ()
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "mips")
if (NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
endif()
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "mips")
if(NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW")
add_compile_options(-flto)
endif ()
endif ()
endif ()
endif ()
endif()
endif()
endif()
endif()
add_library(aaruformat SHARED include/aaruformat/consts.h include/aaruformat/enums.h include/aaru.h include/aaruformat.h
include/aaruformat/decls.h include/aaruformat/structs.h src/identify.c src/open.c include/aaruformat/context.h
src/close.c include/aaruformat/errors.h src/read.c include/aaruformat/crc64.h src/cst.c src/ecc_cd.c src/helpers.c
src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c
src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h
src/flac.c src/lzma.c src/lru.c include/aaruformat/lru.h include/aaruformat/endian.h src/verify.c
include/aaruformat/structs/header.h
include/aaruformat/structs/ddt.h
include/aaruformat/structs/index.h
include/aaruformat/structs/data.h
include/aaruformat/structs/metadata.h
include/aaruformat/structs/dump.h
include/aaruformat/structs/checksum.h
include/aaruformat/structs/optical.h)
include/aaruformat/decls.h include/aaruformat/structs.h src/identify.c src/open.c include/aaruformat/context.h
src/close.c include/aaruformat/errors.h src/read.c include/aaruformat/crc64.h src/cst.c src/ecc_cd.c src/helpers.c
src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c
src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h
src/flac.c src/lzma.c src/lru.c include/aaruformat/lru.h include/aaruformat/endian.h src/verify.c
include/aaruformat/structs/header.h
include/aaruformat/structs/ddt.h
include/aaruformat/structs/index.h
include/aaruformat/structs/data.h
include/aaruformat/structs/metadata.h
include/aaruformat/structs/dump.h
include/aaruformat/structs/checksum.h
include/aaruformat/structs/optical.h
src/index/index_v1.c
include/internal.h
src/index/index_v2.c)
include_directories(include include/aaruformat)
@@ -114,7 +121,7 @@ include(3rdparty/flac.cmake)
include(3rdparty/lzma.cmake)
macro(TARGET_LINK_LIBRARIES_WHOLE_ARCHIVE target)
if (MSVC)
if(MSVC)
foreach(lib IN LISTS ARGN)
set_target_properties(${target} PROPERTIES LINK_FLAGS "/WHOLEARCHIVE:${lib}")
endforeach()
@@ -129,11 +136,11 @@ macro(TARGET_LINK_LIBRARIES_WHOLE_ARCHIVE target)
endif()
endmacro()
if (NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW" OR (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"))
if(NOT "${CMAKE_C_PLATFORM_ID}" MATCHES "MinGW" OR (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"))
set_property(TARGET aaruformat PROPERTY POSITION_INDEPENDENT_CODE TRUE)
else ()
else()
set_property(TARGET aaruformat PROPERTY POSITION_INDEPENDENT_CODE FALSE)
endif ()
endif()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake-modules")
@@ -142,32 +149,32 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake-modules")
find_package(OpenSSL QUIET)
find_package(LibreSSL QUIET)
if (OpenSSL_FOUND)
if(OpenSSL_FOUND)
message("-- OpenSSL VERSION: ${OPENSSL_VERSION}")
endif ()
endif()
if (LIBRESSL_FOUND)
if(LIBRESSL_FOUND)
message("-- LibreSSL VERSION: ${LIBRESSL_VERSION}")
endif ()
endif()
if (OpenSSL_FOUND OR LIBRESSL_FOUND)
if(OpenSSL_FOUND OR LIBRESSL_FOUND)
add_compile_definitions(AARU_HAS_SHA256)
endif ()
endif()
if (LIBRESSL_FOUND)
if(LIBRESSL_FOUND)
TARGET_LINK_LIBRARIES_WHOLE_ARCHIVE(aaruformat ${LIBRESSL_CRYPTO_LIBRARY})
elseif (OpenSSL_FOUND)
elseif(OpenSSL_FOUND)
TARGET_LINK_LIBRARIES_WHOLE_ARCHIVE(aaruformat ${OPENSSL_CRYPTO_LIBRARY})
endif ()
endif()
include_directories(include 3rdparty/uthash/src)
include(CheckLibraryExists)
check_library_exists(m log "" HAVE_LIB_M)
if (HAVE_LIB_M)
if(HAVE_LIB_M)
TARGET_LINK_LIBRARIES_WHOLE_ARCHIVE(aaruformat m)
endif ()
endif()
add_subdirectory(tests)
add_subdirectory(tool)

6
include/aaruformat/enums.h
View File

@@ -205,9 +205,11 @@ typedef enum
/** Block containing data */
DataBlock = 0x4B4C4244,
/** Block containing a deduplication table */
DeDuplicationTable = 0X2A544444,
DeDuplicationTable = 0x2A544444,
/** Block containing the index */
IndexBlock = 0X58444E49,
IndexBlock = 0x58444E49,
/** Block containing the index v2 */
IndexBlock2 = 0x32584449,
/** Block containing logical geometry */
GeometryBlock = 0x4D4F4547,
/** Block containing metadata */

19
include/aaruformat/structs/index.h
View File

@@ -22,7 +22,8 @@
#pragma pack(push, 1)
/**Header for the index, followed by entries */
typedef struct IndexHeader {
typedef struct IndexHeader
{
/**Identifier, <see cref="BlockType.Index" /> */
uint32_t identifier;
/**How many entries follow this header */
@@ -31,8 +32,20 @@ typedef struct IndexHeader {
uint64_t crc64;
} IndexHeader;
/**Header for the index, followed by entries */
typedef struct IndexHeader2
{
/**Identifier, <see cref="BlockType.Index" /> */
uint32_t identifier;
/**How many entries follow this header */
uint64_t entries;
/**CRC64-ECMA of the index */
uint64_t crc64;
} IndexHeader2;
/**Index entry */
typedef struct IndexEntry {
typedef struct IndexEntry
{
/**Type of item pointed by this entry */
uint32_t blockType;
/**Type of data contained by the block pointed by this entry */
@@ -43,4 +56,4 @@ typedef struct IndexEntry {
#pragma pack(pop)
#endif //LIBAARUFORMAT_INDEX_H
#endif // LIBAARUFORMAT_INDEX_H

2
include/internal.h
View File

@@ -23,5 +23,7 @@
UT_array *process_index_v1(aaruformatContext *ctx);
int32_t verify_index_v1(aaruformatContext *ctx);
UT_array *process_index_v2(aaruformatContext *ctx);
int32_t verify_index_v2(aaruformatContext *ctx);
#endif // LIBAARUFORMAT_INTERNAL_H

0
src/index_v1.c → src/index/index_v1.c
View File

120
src/index/index_v2.c Normal file
View File

@@ -0,0 +1,120 @@
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2025 Natalia Portillo.
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "aaruformat.h"
#include "utarray.h"
UT_array *process_index_v2(aaruformatContext *ctx)
{
UT_array *index_entries = NULL;
IndexEntry entry;
if(ctx == NULL || ctx->imageStream == NULL) return NULL;
// Initialize the index entries array
UT_icd index_entry_icd = {sizeof(IndexEntry), NULL, NULL, NULL};
utarray_new(index_entries, &index_entry_icd);
// Read the index header
fseek(ctx->imageStream, ctx->header.indexOffset, SEEK_SET);
IndexHeader2 idx_header;
fread(&idx_header, sizeof(IndexHeader2), 1, ctx->imageStream);
// Check if the index header is valid
if(idx_header.identifier != IndexBlock2)
{
fprintf(stderr, "Incorrect index identifier.\n");
utarray_free(index_entries);
return NULL;
}
for(int i = 0; i < idx_header.entries; i++)
{
fread(&entry, sizeof(IndexEntry), 1, ctx->imageStream);
utarray_push_back(index_entries, &entry);
}
return index_entries;
}
int32_t verify_index_v2(aaruformatContext *ctx)
{
size_t read_bytes = 0;
IndexHeader index_header;
uint64_t crc64 = 0;
IndexEntry *index_entries = NULL;
if(ctx == NULL || ctx->imageStream == NULL) return AARUF_ERROR_NOT_AARUFORMAT;
// This will traverse all blocks and check their CRC64 without uncompressing them
fprintf(stderr, "Checking index integrity at %llu.\n", ctx->header.indexOffset);
fseek(ctx->imageStream, ctx->header.indexOffset, SEEK_SET);
// Read the index header
read_bytes = fread(&index_header, 1, sizeof(IndexHeader2), ctx->imageStream);
if(read_bytes != sizeof(IndexHeader2))
{
fprintf(stderr, "Could not read index header.\n");
return AARUF_ERROR_CANNOT_READ_HEADER;
}
if(index_header.identifier != IndexBlock)
{
fprintf(stderr, "Incorrect index identifier.\n");
return AARUF_ERROR_CANNOT_READ_INDEX;
}
fprintf(stderr, "Index at %llu contains %d entries.\n", ctx->header.indexOffset, index_header.entries);
index_entries = malloc(sizeof(IndexEntry) * index_header.entries);
if(index_entries == NULL)
{
fprintf(stderr, "Cannot allocate memory for index entries.\n");
return AARUF_ERROR_NOT_ENOUGH_MEMORY;
}
read_bytes = fread(index_entries, 1, sizeof(IndexEntry) * index_header.entries, ctx->imageStream);
if(read_bytes != sizeof(IndexEntry) * index_header.entries)
{
fprintf(stderr, "Could not read index entries.\n");
free(index_entries);
return AARUF_ERROR_CANNOT_READ_INDEX;
}
crc64 = aaruf_crc64_data((const uint8_t *)index_entries, sizeof(IndexEntry) * index_header.entries);
// Due to how C# wrote it, it is effectively reversed
if(ctx->header.imageMajorVersion <= AARUF_VERSION) crc64 = bswap_64(crc64);
if(crc64 != index_header.crc64)
{
fprintf(stderr, "Expected index CRC 0x%16llX but got 0x%16llX.\n", index_header.crc64, crc64);
free(index_entries);
return AARUF_ERROR_INVALID_BLOCK_CRC;
}
return AARUF_STATUS_OK;
}

7
src/open.c
View File

@@ -152,7 +152,7 @@ void *aaruf_open(const char *filepath)
readBytes = fread(&signature, 1, sizeof(uint32_t), ctx->imageStream);
if(readBytes != sizeof(uint32_t) || signature != IndexBlock)
if(readBytes != sizeof(uint32_t) || (signature != IndexBlock && signature != IndexBlock2))
{
free(ctx);
errno = AARUF_ERROR_CANNOT_READ_INDEX;
@@ -160,7 +160,10 @@ void *aaruf_open(const char *filepath)
return NULL;
}
index_entries = process_index_v1(ctx);
if(signature == IndexBlock)
index_entries = process_index_v1(ctx);
else if(signature == IndexBlock2)
index_entries = process_index_v2(ctx);
if(index_entries == NULL)
{

12
src/verify.c
View File

@@ -55,14 +55,17 @@ int32_t aaruf_verify_image(void *context)
return AARUF_ERROR_CANNOT_READ_HEADER;
}
if(signature != IndexBlock)
if(signature != IndexBlock && signature != IndexBlock2)
{
fprintf(stderr, "Incorrect index signature.\n");
return AARUF_ERROR_CANNOT_READ_INDEX;
}
// Check if the index is correct
err = verify_index_v1(ctx);
if(signature == IndexBlock)
err = verify_index_v1(ctx);
else if(signature == IndexBlock2)
err = verify_index_v2(ctx);
if(err != AARUF_STATUS_OK)
{
@@ -71,7 +74,10 @@ int32_t aaruf_verify_image(void *context)
}
// Process the index
index_entries = process_index_v1(ctx);
if(signature == IndexBlock)
index_entries = process_index_v1(ctx);
else if(signature == IndexBlock2)
index_entries = process_index_v2(ctx);
if(index_entries == NULL)
{