diff --git a/CMakeLists.txt b/CMakeLists.txt index 961f6b3..b745131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,9 @@ endif() add_library(aaruformat SHARED include/aaruformat/consts.h include/aaruformat/enums.h include/aaru.h include/aaruformat.h include/aaruformat/decls.h include/aaruformat/structs.h src/identify.c src/open.c include/aaruformat/context.h src/close.c include/aaruformat/errors.h src/read.c include/aaruformat/crc64.h src/cst.c src/ecc_cd.c src/helpers.c - src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h src/flac.c src/lzma.c) + src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c + src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h + src/flac.c src/lzma.c src/lru.c include/aaruformat/lru.h) include_directories(include include/aaruformat) diff --git a/README.md b/README.md index 62d191e..240709a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ This means any hash or compression algorithm must be statically linked inside th cmake is not a hard dependency, it's merely for the ease of using IDEs (specifically CLion). Things still to be implemented that are already in the C# version: -- Caching - Tape file blocks - Automatic media type generation from C# enumeration - Nuget package for linking with Aaru diff --git a/include/aaruformat.h b/include/aaruformat.h index fcf7136..b9acd1a 100644 --- a/include/aaruformat.h +++ b/include/aaruformat.h @@ -28,6 +28,7 @@ #include "aaruformat/decls.h" #include "aaruformat/enums.h" #include "aaruformat/errors.h" +#include "aaruformat/lru.h" #include "aaruformat/simd.h" #include "aaruformat/spamsum.h" #include "aaruformat/structs.h" diff --git a/include/aaruformat/consts.h b/include/aaruformat/consts.h index b73513e..62c69d2 100644 --- a/include/aaruformat/consts.h +++ b/include/aaruformat/consts.h @@ -31,8 +31,8 @@ /** Image format version. A change in this number indicates an incompatible change to the format that prevents older * implementations from reading it correctly, if at all. */ #define AARUF_VERSION 1 -/** Maximum read cache size, 256MiB. */ -#define MAX_CACHE_SIZE 256 * 1024 * 1024 +/** Maximum read cache size, 512MiB. */ +#define MAX_CACHE_SIZE 536870912 /** Size in bytes of LZMA properties. */ #define LZMA_PROPERTIES_LENGTH 5 /** Maximum number of entries for the DDT cache. */ diff --git a/include/aaruformat/context.h b/include/aaruformat/context.h index 2ccc20e..ce4434f 100644 --- a/include/aaruformat/context.h +++ b/include/aaruformat/context.h @@ -19,6 +19,7 @@ #ifndef LIBAARUFORMAT_CONTEXT_H #define LIBAARUFORMAT_CONTEXT_H +#include "lru.h" #include "structs.h" typedef struct aaruformatContext @@ -56,6 +57,8 @@ typedef struct aaruformatContext uint8_t numberOfDataTracks; TrackEntry* dataTracks; bool* readableSectorTags; + struct CacheHeader blockHeaderCache; + struct CacheHeader blockCache; } aaruformatContext; typedef struct dataLinkedList diff --git a/include/aaruformat/lru.h b/include/aaruformat/lru.h new file mode 100644 index 0000000..b4696b7 --- /dev/null +++ b/include/aaruformat/lru.h @@ -0,0 +1,56 @@ +// +// Created by claunia on 2/10/22. +// + +#ifndef LIBAARUFORMAT_LRU_H +#define LIBAARUFORMAT_LRU_H + +#include +#include + +struct CacheEntry +{ + char* key; + void* value; + UT_hash_handle hh; +}; + +struct CacheHeader +{ + uint64_t max_items; + struct CacheEntry* cache; +}; + +/** + * Finds an item in the specified cache + * @param cache Pointer to the cache header + * @param key Key + * @return Value if found, NULL if not + */ +void* find_in_cache(struct CacheHeader* cache, char* key); + +/** + * Adds an item to the specified cache + * @param cache Pointer to the cache header + * @param key Key + * @param value Value + */ +void add_to_cache(struct CacheHeader* cache, char* key, void* value); + +/** + * Finds an item in the specified cache using a 64-bit integer key + * @param cache Pointer to the cache header + * @param key Key + * @return Value if found, NULL if not + */ +void* find_in_cache_uint64(struct CacheHeader* cache, uint64_t key); + +/** + * Adds an item to the specified cache using a 64-bit integer key + * @param cache Pointer to the cache header + * @param key Key + * @param value Value + */ +void add_to_cache_uint64(struct CacheHeader* cache, uint64_t key, void* value); + +#endif // LIBAARUFORMAT_LRU_H diff --git a/src/lru.c b/src/lru.c new file mode 100644 index 0000000..0b78e55 --- /dev/null +++ b/src/lru.c @@ -0,0 +1,83 @@ +#include +#include + +#include + +// this is an example of how to do a LRU cache in C using uthash +// http://uthash.sourceforge.net/ +// by Jehiah Czebotar 2011 - jehiah@gmail.com +// this code is in the public domain http://unlicense.org/ + +void* find_in_cache(struct CacheHeader* cache, char* key) +{ + struct CacheEntry* entry; + HASH_FIND_STR(cache->cache, key, entry); + if(entry) + { + // remove it (so the subsequent add will throw it on the front of the list) + HASH_DELETE(hh, cache->cache, entry); + HASH_ADD_KEYPTR(hh, cache->cache, entry->key, strlen(entry->key), entry); + return entry->value; + } + return NULL; +} + +void add_to_cache(struct CacheHeader* cache, char* key, void* value) +{ + struct CacheEntry *entry, *tmp_entry; + // TODO: Is this needed or we're just losing cycles? uthash does not free the entry + entry = malloc(sizeof(struct CacheEntry)); + entry->key = strdup(key); + entry->value = value; + HASH_ADD_KEYPTR(hh, cache->cache, entry->key, strlen(entry->key), entry); + + // prune the cache to MAX_CACHE_SIZE + if(HASH_COUNT(cache->cache) >= cache->max_items) + { + HASH_ITER(hh, cache->cache, entry, tmp_entry) + { + // prune the first entry (loop is based on insertion order so this deletes the oldest item) + HASH_DELETE(hh, cache->cache, entry); + free(entry->key); + free(entry); + break; + } + } +} + +FORCE_INLINE char* int64_to_string(uint64_t number) +{ + char* charKey; + + charKey = malloc(17); + + charKey[0] = (char)(number >> 60); + charKey[1] = (char)((number & 0xF00000000000000ULL) >> 56); + charKey[2] = (char)((number & 0xF0000000000000ULL) >> 52); + charKey[3] = (char)((number & 0xF000000000000ULL) >> 48); + charKey[4] = (char)((number & 0xF00000000000ULL) >> 44); + charKey[5] = (char)((number & 0xF0000000000ULL) >> 40); + charKey[6] = (char)((number & 0xF000000000ULL) >> 36); + charKey[7] = (char)((number & 0xF00000000ULL) >> 32); + charKey[8] = (char)((number & 0xF0000000ULL) >> 28); + charKey[9] = (char)((number & 0xF000000ULL) >> 24); + charKey[10] = (char)((number & 0xF00000ULL) >> 20); + charKey[11] = (char)((number & 0xF0000ULL) >> 16); + charKey[12] = (char)((number & 0xF000ULL) >> 12); + charKey[13] = (char)((number & 0xF00ULL) >> 8); + charKey[14] = (char)((number & 0xF0ULL) >> 4); + charKey[15] = (char)(number & 0xFULL); + charKey[16] = 0; + + return charKey; +} + +void* find_in_cache_uint64(struct CacheHeader* cache, uint64_t key) +{ + return find_in_cache(cache, int64_to_string(key)); +} + +void add_to_cache_uint64(struct CacheHeader* cache, uint64_t key, void* value) +{ + return add_to_cache(cache, int64_to_string(key), value); +} \ No newline at end of file diff --git a/src/open.c b/src/open.c index 625233a..6bf5ff0 100644 --- a/src/open.c +++ b/src/open.c @@ -421,8 +421,7 @@ void* aaruf_open(const char* filepath) ctx->inMemoryDdt = false; break; #else // TODO: Implement - fprintf(stderr, - "libaaruformat: Uncompressed DDT not yet implemented..."); + fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented..."); foundUserDataDdt = false; break; #endif @@ -754,8 +753,8 @@ void* aaruf_open(const char* filepath) fprintf(stderr, "libaaruformat: Could not read metadata block, continuing..."); } - crc64 = aaruf_crc64_data((const uint8_t*)ctx->trackEntries, - ctx->tracksHeader.entries * sizeof(TrackEntry)); + crc64 = + aaruf_crc64_data((const uint8_t*)ctx->trackEntries, ctx->tracksHeader.entries * sizeof(TrackEntry)); if(crc64 != ctx->tracksHeader.crc64) { fprintf(stderr, @@ -1153,14 +1152,11 @@ void* aaruf_open(const char* filepath) ctx->imageInfo.SectorsPerTrack = 63; } - // TODO: Caches - /* - // Initialize caches - blockCache = new Dictionary(); - blockHeaderCache = new Dictionary(); - currentCacheSize = 0; - if(!inMemoryDdt) ddtEntryCache = new Dictionary(); - */ + // Initialize caches + ctx->blockHeaderCache.cache = NULL; + ctx->blockHeaderCache.max_items = MAX_CACHE_SIZE / (ctx->imageInfo.SectorSize * (1 << ctx->shift)); + ctx->blockCache.cache = NULL; + ctx->blockCache.max_items = ctx->blockHeaderCache.max_items; // TODO: Cache tracks and sessions? diff --git a/src/read.c b/src/read.c index 53be2e7..f0c681c 100644 --- a/src/read.c +++ b/src/read.c @@ -63,7 +63,7 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data, uint32_t offsetMask; uint64_t offset; uint64_t blockOffset; - BlockHeader blockHeader; + BlockHeader* blockHeader; uint8_t* block; size_t readBytes; @@ -89,31 +89,51 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data, return AARUF_STATUS_SECTOR_NOT_DUMPED; } - // Check if block is cached - // TODO: Caches + // Check if block header is cached + blockHeader = find_in_cache_uint64(&ctx->blockHeaderCache, blockOffset); // Read block header - fseek(ctx->imageStream, blockOffset, SEEK_SET); - readBytes = fread(&blockHeader, sizeof(BlockHeader), 1, ctx->imageStream); - - if(readBytes != sizeof(BlockHeader)) return AARUF_ERROR_CANNOT_READ_HEADER; - - if(data == NULL || *length < blockHeader.sectorSize) + if(blockHeader == NULL) { - *length = blockHeader.sectorSize; + blockHeader = malloc(sizeof(BlockHeader)); + if(blockHeader == NULL) return AARUF_ERROR_NOT_ENOUGH_MEMORY; + + fseek(ctx->imageStream, blockOffset, SEEK_SET); + readBytes = fread(blockHeader, sizeof(BlockHeader), 1, ctx->imageStream); + + if(readBytes != sizeof(BlockHeader)) return AARUF_ERROR_CANNOT_READ_HEADER; + + add_to_cache_uint64(&ctx->blockHeaderCache, blockOffset, blockHeader); + } + else + fseek(ctx->imageStream, sizeof(BlockHeader), SEEK_CUR); // Advance as if reading the header + + if(data == NULL || *length < blockHeader->sectorSize) + { + *length = blockHeader->sectorSize; return AARUF_ERROR_BUFFER_TOO_SMALL; } + // Check if block is cached + block = find_in_cache_uint64(&ctx->blockCache, blockOffset); + + if(block != NULL) + { + memcpy(data, block + offset, blockHeader->sectorSize); + *length = blockHeader->sectorSize; + return AARUF_STATUS_OK; + } + // Decompress block - switch(blockHeader.compression) + switch(blockHeader->compression) { case None: - block = (uint8_t*)malloc(blockHeader.length); + block = (uint8_t*)malloc(blockHeader->length); if(block == NULL) return AARUF_ERROR_NOT_ENOUGH_MEMORY; - readBytes = fread(block, blockHeader.length, 1, ctx->imageStream); + readBytes = fread(block, blockHeader->length, 1, ctx->imageStream); - if(readBytes != blockHeader.length) + if(readBytes != blockHeader->length) { free(block); return AARUF_ERROR_CANNOT_READ_BLOCK; @@ -123,15 +143,11 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data, default: return AARUF_ERROR_UNSUPPORTED_COMPRESSION; } - // Check if cache needs to be emptied - // TODO: Caches - // Add block to cache - // TODO: Caches + add_to_cache_uint64(&ctx->blockCache, blockOffset, block); - memcpy(data, block + offset, blockHeader.sectorSize); - *length = blockHeader.sectorSize; - free(block); + memcpy(data, block + offset, blockHeader->sectorSize); + *length = blockHeader->sectorSize; return AARUF_STATUS_OK; }