Implement caches.

CMakeLists.txt

@@ -96,7 +96,9 @@ endif()
 add_library(aaruformat SHARED include/aaruformat/consts.h include/aaruformat/enums.h include/aaru.h include/aaruformat.h
             include/aaruformat/decls.h include/aaruformat/structs.h src/identify.c src/open.c include/aaruformat/context.h
             src/close.c include/aaruformat/errors.h src/read.c include/aaruformat/crc64.h src/cst.c src/ecc_cd.c src/helpers.c
-            src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h src/flac.c src/lzma.c)
+            src/simd.c include/aaruformat/simd.h src/crc64/crc64.c src/crc64/crc64_clmul.c src/crc64/crc64_vmull.c
+            src/crc64/arm_vmull.c src/crc64/arm_vmull.h src/spamsum.c include/aaruformat/spamsum.h include/aaruformat/flac.h
+            src/flac.c src/lzma.c src/lru.c include/aaruformat/lru.h)
 
 include_directories(include include/aaruformat)
 

README.md

@@ -9,7 +9,6 @@ This means any hash or compression algorithm must be statically linked inside th
 cmake is not a hard dependency, it's merely for the ease of using IDEs (specifically CLion).
 
 Things still to be implemented that are already in the C# version:
-- Caching
 - Tape file blocks
 - Automatic media type generation from C# enumeration
 - Nuget package for linking with Aaru

include/aaruformat.h

@@ -28,6 +28,7 @@
 #include "aaruformat/decls.h"
 #include "aaruformat/enums.h"
 #include "aaruformat/errors.h"
+#include "aaruformat/lru.h"
 #include "aaruformat/simd.h"
 #include "aaruformat/spamsum.h"
 #include "aaruformat/structs.h"

include/aaruformat/consts.h

@@ -31,8 +31,8 @@
 /** Image format version. A change in this number indicates an incompatible change to the format that prevents older
  * implementations from reading it correctly, if at all. */
 #define AARUF_VERSION 1
-/** Maximum read cache size, 256MiB. */
+/** Maximum read cache size, 512MiB. */
-#define MAX_CACHE_SIZE 256 * 1024 * 1024
+#define MAX_CACHE_SIZE 536870912
 /** Size in bytes of LZMA properties. */
 #define LZMA_PROPERTIES_LENGTH 5
 /** Maximum number of entries for the DDT cache. */

include/aaruformat/context.h

@@ -19,6 +19,7 @@
 #ifndef LIBAARUFORMAT_CONTEXT_H
 #define LIBAARUFORMAT_CONTEXT_H
 
+#include "lru.h"
 #include "structs.h"
 
 typedef struct aaruformatContext
@@ -56,6 +57,8 @@ typedef struct aaruformatContext
     uint8_t                             numberOfDataTracks;
     TrackEntry*                         dataTracks;
     bool*                               readableSectorTags;
+    struct CacheHeader                  blockHeaderCache;
+    struct CacheHeader                  blockCache;
 } aaruformatContext;
 
 typedef struct dataLinkedList

include/aaruformat/lru.h

@@ -0,0 +1,56 @@
+//
+// Created by claunia on 2/10/22.
+//
+
+#ifndef LIBAARUFORMAT_LRU_H
+#define LIBAARUFORMAT_LRU_H
+
+#include <stdint.h>
+#include <uthash.h>
+
+struct CacheEntry
+{
+    char*          key;
+    void*          value;
+    UT_hash_handle hh;
+};
+
+struct CacheHeader
+{
+    uint64_t           max_items;
+    struct CacheEntry* cache;
+};
+
+/**
+ * Finds an item in the specified cache
+ * @param cache Pointer to the cache header
+ * @param key Key
+ * @return Value if found, NULL if not
+ */
+void* find_in_cache(struct CacheHeader* cache, char* key);
+
+/**
+ * Adds an item to the specified cache
+ * @param cache Pointer to the cache header
+ * @param key Key
+ * @param value Value
+ */
+void add_to_cache(struct CacheHeader* cache, char* key, void* value);
+
+/**
+ * Finds an item in the specified cache using a 64-bit integer key
+ * @param cache Pointer to the cache header
+ * @param key Key
+ * @return Value if found, NULL if not
+ */
+void* find_in_cache_uint64(struct CacheHeader* cache, uint64_t key);
+
+/**
+ * Adds an item to the specified cache using a 64-bit integer key
+ * @param cache Pointer to the cache header
+ * @param key Key
+ * @param value Value
+ */
+void add_to_cache_uint64(struct CacheHeader* cache, uint64_t key, void* value);
+
+#endif // LIBAARUFORMAT_LRU_H

src/lru.c

@@ -0,0 +1,83 @@
+#include <string.h>
+#include <uthash.h>
+
+#include <aaruformat.h>
+
+// this is an example of how to do a LRU cache in C using uthash
+// http://uthash.sourceforge.net/
+// by Jehiah Czebotar 2011 - jehiah@gmail.com
+// this code is in the public domain http://unlicense.org/
+
+void* find_in_cache(struct CacheHeader* cache, char* key)
+{
+    struct CacheEntry* entry;
+    HASH_FIND_STR(cache->cache, key, entry);
+    if(entry)
+    {
+        // remove it (so the subsequent add will throw it on the front of the list)
+        HASH_DELETE(hh, cache->cache, entry);
+        HASH_ADD_KEYPTR(hh, cache->cache, entry->key, strlen(entry->key), entry);
+        return entry->value;
+    }
+    return NULL;
+}
+
+void add_to_cache(struct CacheHeader* cache, char* key, void* value)
+{
+    struct CacheEntry *entry, *tmp_entry;
+    // TODO: Is this needed or we're just losing cycles? uthash does not free the entry
+    entry        = malloc(sizeof(struct CacheEntry));
+    entry->key   = strdup(key);
+    entry->value = value;
+    HASH_ADD_KEYPTR(hh, cache->cache, entry->key, strlen(entry->key), entry);
+
+    // prune the cache to MAX_CACHE_SIZE
+    if(HASH_COUNT(cache->cache) >= cache->max_items)
+    {
+        HASH_ITER(hh, cache->cache, entry, tmp_entry)
+        {
+            // prune the first entry (loop is based on insertion order so this deletes the oldest item)
+            HASH_DELETE(hh, cache->cache, entry);
+            free(entry->key);
+            free(entry);
+            break;
+        }
+    }
+}
+
+FORCE_INLINE char* int64_to_string(uint64_t number)
+{
+    char* charKey;
+
+    charKey = malloc(17);
+
+    charKey[0]  = (char)(number >> 60);
+    charKey[1]  = (char)((number & 0xF00000000000000ULL) >> 56);
+    charKey[2]  = (char)((number & 0xF0000000000000ULL) >> 52);
+    charKey[3]  = (char)((number & 0xF000000000000ULL) >> 48);
+    charKey[4]  = (char)((number & 0xF00000000000ULL) >> 44);
+    charKey[5]  = (char)((number & 0xF0000000000ULL) >> 40);
+    charKey[6]  = (char)((number & 0xF000000000ULL) >> 36);
+    charKey[7]  = (char)((number & 0xF00000000ULL) >> 32);
+    charKey[8]  = (char)((number & 0xF0000000ULL) >> 28);
+    charKey[9]  = (char)((number & 0xF000000ULL) >> 24);
+    charKey[10] = (char)((number & 0xF00000ULL) >> 20);
+    charKey[11] = (char)((number & 0xF0000ULL) >> 16);
+    charKey[12] = (char)((number & 0xF000ULL) >> 12);
+    charKey[13] = (char)((number & 0xF00ULL) >> 8);
+    charKey[14] = (char)((number & 0xF0ULL) >> 4);
+    charKey[15] = (char)(number & 0xFULL);
+    charKey[16] = 0;
+
+    return charKey;
+}
+
+void* find_in_cache_uint64(struct CacheHeader* cache, uint64_t key)
+{
+    return find_in_cache(cache, int64_to_string(key));
+}
+
+void add_to_cache_uint64(struct CacheHeader* cache, uint64_t key, void* value)
+{
+    return add_to_cache(cache, int64_to_string(key), value);
+}

src/open.c

@@ -421,8 +421,7 @@ void* aaruf_open(const char* filepath)
                             ctx->inMemoryDdt = false;
                             break;
 #else // TODO: Implement
-                            fprintf(stderr,
+                            fprintf(stderr, "libaaruformat: Uncompressed DDT not yet implemented...");
-                                    "libaaruformat: Uncompressed DDT not yet implemented...");
                             foundUserDataDdt = false;
                             break;
 #endif
@@ -754,8 +753,8 @@ void* aaruf_open(const char* filepath)
                     fprintf(stderr, "libaaruformat: Could not read metadata block, continuing...");
                 }
 
-                crc64 = aaruf_crc64_data((const uint8_t*)ctx->trackEntries,
+                crc64 =
-                                              ctx->tracksHeader.entries * sizeof(TrackEntry));
+                    aaruf_crc64_data((const uint8_t*)ctx->trackEntries, ctx->tracksHeader.entries * sizeof(TrackEntry));
                 if(crc64 != ctx->tracksHeader.crc64)
                 {
                     fprintf(stderr,
@@ -1153,14 +1152,11 @@ void* aaruf_open(const char* filepath)
         ctx->imageInfo.SectorsPerTrack = 63;
     }
 
-    // TODO: Caches
-    /*
     // Initialize caches
-        blockCache       = new Dictionary<ulong, byte[]>();
+    ctx->blockHeaderCache.cache     = NULL;
-        blockHeaderCache = new Dictionary<ulong, BlockHeader>();
+    ctx->blockHeaderCache.max_items = MAX_CACHE_SIZE / (ctx->imageInfo.SectorSize * (1 << ctx->shift));
-        currentCacheSize = 0;
+    ctx->blockCache.cache           = NULL;
-        if(!inMemoryDdt) ddtEntryCache = new Dictionary<ulong, ulong>();
+    ctx->blockCache.max_items       = ctx->blockHeaderCache.max_items;
-    */
 
     // TODO: Cache tracks and sessions?
 

src/read.c

@@ -63,7 +63,7 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data,
     uint32_t           offsetMask;
     uint64_t           offset;
     uint64_t           blockOffset;
-    BlockHeader        blockHeader;
+    BlockHeader*       blockHeader;
     uint8_t*           block;
     size_t             readBytes;
 
@@ -89,31 +89,51 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data,
         return AARUF_STATUS_SECTOR_NOT_DUMPED;
     }
 
-    // Check if block is cached
+    // Check if block header is cached
-    // TODO: Caches
+    blockHeader = find_in_cache_uint64(&ctx->blockHeaderCache, blockOffset);
 
     // Read block header
+    if(blockHeader == NULL)
+    {
+        blockHeader = malloc(sizeof(BlockHeader));
+        if(blockHeader == NULL) return AARUF_ERROR_NOT_ENOUGH_MEMORY;
+
         fseek(ctx->imageStream, blockOffset, SEEK_SET);
-    readBytes = fread(&blockHeader, sizeof(BlockHeader), 1, ctx->imageStream);
+        readBytes = fread(blockHeader, sizeof(BlockHeader), 1, ctx->imageStream);
 
         if(readBytes != sizeof(BlockHeader)) return AARUF_ERROR_CANNOT_READ_HEADER;
 
-    if(data == NULL || *length < blockHeader.sectorSize)
+        add_to_cache_uint64(&ctx->blockHeaderCache, blockOffset, blockHeader);
+    }
+    else
+        fseek(ctx->imageStream, sizeof(BlockHeader), SEEK_CUR); // Advance as if reading the header
+
+    if(data == NULL || *length < blockHeader->sectorSize)
     {
-        *length = blockHeader.sectorSize;
+        *length = blockHeader->sectorSize;
         return AARUF_ERROR_BUFFER_TOO_SMALL;
     }
 
+    // Check if block is cached
+    block = find_in_cache_uint64(&ctx->blockCache, blockOffset);
+
+    if(block != NULL)
+    {
+        memcpy(data, block + offset, blockHeader->sectorSize);
+        *length = blockHeader->sectorSize;
+        return AARUF_STATUS_OK;
+    }
+
     // Decompress block
-    switch(blockHeader.compression)
+    switch(blockHeader->compression)
     {
         case None:
-            block = (uint8_t*)malloc(blockHeader.length);
+            block = (uint8_t*)malloc(blockHeader->length);
             if(block == NULL) return AARUF_ERROR_NOT_ENOUGH_MEMORY;
 
-            readBytes = fread(block, blockHeader.length, 1, ctx->imageStream);
+            readBytes = fread(block, blockHeader->length, 1, ctx->imageStream);
 
-            if(readBytes != blockHeader.length)
+            if(readBytes != blockHeader->length)
             {
                 free(block);
                 return AARUF_ERROR_CANNOT_READ_BLOCK;
@@ -123,15 +143,11 @@ int32_t aaruf_read_sector(void* context, uint64_t sectorAddress, uint8_t* data,
         default: return AARUF_ERROR_UNSUPPORTED_COMPRESSION;
     }
 
-    // Check if cache needs to be emptied
-    // TODO: Caches
-
     // Add block to cache
-    // TODO: Caches
+    add_to_cache_uint64(&ctx->blockCache, blockOffset, block);
 
-    memcpy(data, block + offset, blockHeader.sectorSize);
+    memcpy(data, block + offset, blockHeader->sectorSize);
-    *length = blockHeader.sectorSize;
+    *length = blockHeader->sectorSize;
-    free(block);
     return AARUF_STATUS_OK;
 }