Add SpamSum.

spamsum.c

@@ -0,0 +1,312 @@
+/*
+ * This file is part of the Aaru Data Preservation Suite.
+ * Copyright (c) 2019-2021 Natalia Portillo.
+ *
+ * This library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of the
+ * License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "spamsum.h"
+
+#include "library.h"
+
+static uint8_t _b64[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D,
+                         0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A,
+                         0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D,
+                         0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A,
+                         0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2F};
+
+spamsum_ctx* spamsum_init(void)
+{
+    spamsum_ctx* ctx = (spamsum_ctx*)malloc(sizeof(spamsum_ctx));
+    if(!ctx) return NULL;
+
+    memset(ctx, 0, sizeof(spamsum_ctx));
+
+    ctx->Bhend       = 1;
+    ctx->Bh[0].H     = HASH_INIT;
+    ctx->Bh[0].Halfh = HASH_INIT;
+
+    return ctx;
+}
+
+int spamsum_update(spamsum_ctx* ctx, const uint8_t* data, uint32_t len)
+{
+    if(!ctx || !data) return -1;
+
+    for(int i = 0; i < len; i++) fuzzy_engine_step(ctx, data[i]);
+
+    ctx->TotalSize += len;
+
+    return 0;
+}
+
+void spamsum_free(spamsum_ctx* ctx)
+{
+    if(ctx) free(ctx);
+}
+
+#define roll_sum(ctx) (ctx->Roll.H1 + ctx->Roll.H2 + ctx->Roll.H3)
+#define sum_hash(c, h) ((h * HASH_PRIME) ^ c);
+#define SSDEEP_BS(index) (MIN_BLOCKSIZE << index)
+
+void fuzzy_engine_step(spamsum_ctx* ctx, uint8_t c)
+{
+    uint32_t i;
+    /* At each character we update the rolling hash and the normal hashes.
+     * When the rolling hash hits a reset value then we emit a normal hash
+     * as a element of the signature and reset the normal hash. */
+    roll_hash(ctx, c);
+    uint64_t h = roll_sum(ctx);
+
+    for(i = ctx->Bhstart; i < ctx->Bhend; ++i)
+    {
+        ctx->Bh[i].H     = sum_hash(c, ctx->Bh[i].H);
+        ctx->Bh[i].Halfh = sum_hash(c, ctx->Bh[i].Halfh);
+    }
+
+    for(i = ctx->Bhstart; i < ctx->Bhend; ++i)
+    {
+        /* With growing blocksize almost no runs fail the next test. */
+        if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
+            /* Once this condition is false for one bs, it is
+             * automatically false for all further bs. I.e. if
+             * h === -1 (mod 2*bs) then h === -1 (mod bs). */
+            break;
+
+        /* We have hit a reset point. We now emit hashes which are
+         * based on all characters in the piece of the message between
+         * the last reset point and this one */
+        if(0 == ctx->Bh[i].Dlen) fuzzy_try_fork_blockhash(ctx);
+
+        ctx->Bh[i].Digest[ctx->Bh[i].Dlen] = _b64[ctx->Bh[i].H % 64];
+        ctx->Bh[i].Halfdigest              = _b64[ctx->Bh[i].Halfh % 64];
+
+        if(ctx->Bh[i].Dlen < SPAMSUM_LENGTH - 1)
+        {
+            /* We can have a problem with the tail overflowing. The
+             * easiest way to cope with this is to only reset the
+             * normal hash if we have room for more characters in
+             * our signature. This has the effect of combining the
+             * last few pieces of the message into a single piece
+             * */
+            ctx->Bh[i].Digest[++ctx->Bh[i].Dlen] = 0;
+            ctx->Bh[i].H                         = HASH_INIT;
+
+            if(ctx->Bh[i].Dlen >= SPAMSUM_LENGTH / 2) continue;
+
+            ctx->Bh[i].Halfh      = HASH_INIT;
+            ctx->Bh[i].Halfdigest = 0;
+        }
+        else
+            fuzzy_try_reduce_blockhash(ctx);
+    }
+}
+
+void roll_hash(spamsum_ctx* ctx, uint8_t c)
+{
+    ctx->Roll.H2 -= ctx->Roll.H1;
+    ctx->Roll.H2 += ROLLING_WINDOW * c;
+
+    ctx->Roll.H1 += c;
+    ctx->Roll.H1 -= ctx->Roll.Window[ctx->Roll.N % ROLLING_WINDOW];
+
+    ctx->Roll.Window[ctx->Roll.N % ROLLING_WINDOW] = c;
+    ctx->Roll.N++;
+
+    /* The original spamsum AND'ed this value with 0xFFFFFFFF which
+     * in theory should have no effect. This AND has been removed
+     * for performance (jk) */
+    ctx->Roll.H3 <<= 5;
+    ctx->Roll.H3 ^= c;
+}
+
+void fuzzy_try_reduce_blockhash(spamsum_ctx* ctx)
+{
+    assert(ctx->Bhstart < ctx->Bhend);
+
+    if(ctx->Bhend - ctx->Bhstart < 2) /* Need at least two working hashes. */
+        return;
+
+    if((uint64_t)SSDEEP_BS(ctx->Bhstart) * SPAMSUM_LENGTH >= ctx->TotalSize)
+        /* Initial blocksize estimate would select this or a smaller
+         * blocksize. */
+        return;
+
+    if(ctx->Bh[ctx->Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2) /* Estimate adjustment would select this blocksize. */
+        return;
+
+    /* At this point we are clearly no longer interested in the
+     * start_blocksize. Get rid of it. */
+    ++ctx->Bhstart;
+}
+
+void fuzzy_try_fork_blockhash(spamsum_ctx* ctx)
+{
+    if(ctx->Bhend >= NUM_BLOCKHASHES) return;
+
+    assert(ctx->Bhend != 0);
+
+    uint32_t obh            = ctx->Bhend - 1;
+    uint32_t nbh            = ctx->Bhend;
+    ctx->Bh[nbh].H          = ctx->Bh[obh].H;
+    ctx->Bh[nbh].Halfh      = ctx->Bh[obh].Halfh;
+    ctx->Bh[nbh].Digest[0]  = 0;
+    ctx->Bh[nbh].Halfdigest = 0;
+    ctx->Bh[nbh].Dlen       = 0;
+    ++ctx->Bhend;
+}
+
+uint8_t* spamsum_final(spamsum_ctx* ctx)
+{
+    uint32_t bi     = ctx->Bhstart;
+    uint32_t h      = roll_sum(ctx);
+    int      remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */
+    uint8_t* result = (uint8_t*)malloc(FUZZY_MAX_RESULT);
+
+    if(!result) return NULL;
+
+    /* Verify that our elimination was not overeager. */
+    assert(bi == 0 || (uint64_t)SSDEEP_BS(bi) / 2 * SPAMSUM_LENGTH < ctx->TotalSize);
+
+    /* Initial blocksize guess. */
+    while((uint64_t)SSDEEP_BS(bi) * SPAMSUM_LENGTH < ctx->TotalSize)
+    {
+        ++bi;
+
+        if(bi >= NUM_BLOCKHASHES)
+        {
+            errno = EOVERFLOW;
+            return NULL;
+        }
+    }
+
+    /* Adapt blocksize guess to actual digest length. */
+    while(bi >= ctx->Bhend) --bi;
+
+    while(bi > ctx->Bhstart && ctx->Bh[bi].Dlen < SPAMSUM_LENGTH / 2) --bi;
+
+    assert(!(bi > 0 && ctx->Bh[bi].Dlen < SPAMSUM_LENGTH / 2));
+
+    int i = snprintf((char*)result, (size_t)remain, "%lu:", (unsigned long)SSDEEP_BS(bi));
+
+    if(i <= 0) /* Maybe snprintf has set errno here? */
+        return NULL;
+
+    assert(i < remain);
+
+    remain -= i;
+    result += i;
+
+    i = (int)ctx->Bh[bi].Dlen;
+
+    assert(i <= remain);
+
+    memcpy(result, ctx->Bh[bi].Digest, (size_t)i);
+    result += i;
+    remain -= i;
+
+    if(h != 0)
+    {
+        assert(remain > 0);
+
+        *result = _b64[ctx->Bh[bi].H % 64];
+
+        if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
+        {
+            ++result;
+            --remain;
+        }
+    }
+    else if(ctx->Bh[bi].Digest[i] != 0)
+    {
+        assert(remain > 0);
+
+        *result = ctx->Bh[bi].Digest[i];
+
+        if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
+        {
+            ++result;
+            --remain;
+        }
+    }
+
+    assert(remain > 0);
+
+    *result++ = ':';
+    --remain;
+
+    if(bi < ctx->Bhend - 1)
+    {
+        ++bi;
+        i = (int)ctx->Bh[bi].Dlen;
+
+        if(i <= remain)
+            ;
+
+        memcpy(result, ctx->Bh[bi].Digest, (size_t)i);
+        result += i;
+        remain -= i;
+
+        if(h != 0)
+        {
+            assert(remain > 0);
+
+            h       = ctx->Bh[bi].Halfh;
+            *result = _b64[h % 64];
+
+            if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
+            {
+                ++result;
+                --remain;
+            }
+        }
+        else
+        {
+            i = ctx->Bh[bi].Halfdigest;
+
+            if(i != 0)
+            {
+                assert(remain > 0);
+
+                *result = (uint8_t)i;
+
+                if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
+                {
+                    ++result;
+                    --remain;
+                }
+            }
+        }
+    }
+    else if(h != 0)
+    {
+        assert(ctx->Bh[bi].Dlen == 0);
+
+        assert(remain > 0);
+
+        *result++ = _b64[ctx->Bh[bi].H % 64];
+        /* No need to bother with FUZZY_FLAG_ELIMSEQ, because this
+         * digest has length 1. */
+        --remain;
+    }
+
+    *result = 0;
+
+    return result;
+}