Loading...
Searching...
No Matches
spamsum.c
Go to the documentation of this file.
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2026 Natalia Portillo.
* Copyright (C) 2002 Andrew Tridgell <tridge@samba.org>
* Copyright (C) 2006 ManTech International Corporation
* Copyright (C) 2013 Helmut Grohne <helmut@subdivi.de>
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "aaruformat.h"
#include "spamsum.h"
static uint8_t b64[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76,
0x77, 0x78, 0x79, 0x7A, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2F};
AARU_EXPORT spamsum_ctx *AARU_CALL aaruf_spamsum_init(void)
{
spamsum_ctx *ctx = malloc(sizeof(spamsum_ctx));
if(!ctx) return NULL;
memset(ctx, 0, sizeof(spamsum_ctx));
ctx->bh_end = 1;
ctx->bh[0].h = HASH_INIT;
ctx->bh[0].half_h = HASH_INIT;
return ctx;
}
AARU_EXPORT int AARU_CALL aaruf_spamsum_update(spamsum_ctx *ctx, const uint8_t *data, const uint32_t len)
{
if(!ctx || !data) return -1;
for(int i = 0; i < len; i++) fuzzy_engine_step(ctx, data[i]);
ctx->total_size += len;
return 0;
}
AARU_EXPORT void AARU_CALL aaruf_spamsum_free(spamsum_ctx *ctx)
{
if(ctx) free(ctx);
}
#define ROLL_SUM(ctx) ((ctx)->roll.h1 + (ctx)->roll.h2 + (ctx)->roll.h3)
#define SUM_HASH(c, h) (((h) * HASH_PRIME) ^ (c));
#define SSDEEP_BS(index) (MIN_BLOCKSIZE << (index))
AARU_LOCAL inline void fuzzy_engine_step(spamsum_ctx *ctx, uint8_t c)
{
uint32_t i = 0;
/* At each character we update the rolling hash and the normal hashes.
* When the rolling hash hits a reset value then we emit a normal hash
* as a element of the signature and reset the normal hash. */
roll_hash(ctx, c);
const uint64_t h = ROLL_SUM(ctx);
for(i = ctx->bh_start; i < ctx->bh_end; ++i)
{
ctx->bh[i].h = SUM_HASH(c, ctx->bh[i].h);
ctx->bh[i].half_h = SUM_HASH(c, ctx->bh[i].half_h);
}
for(i = ctx->bh_start; i < ctx->bh_end; ++i)
{
/* With growing blocksize almost no runs fail the next test. */
if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
/* Once this condition is false for one bs, it is
* automatically false for all further bs. I.e. if
* h === -1 (mod 2*bs) then h === -1 (mod bs). */
break;
/* We have hit a reset point. We now emit hashes which are
* based on all characters in the piece of the message between
* the last reset point and this one */
if(0 == ctx->bh[i].d_len) fuzzy_try_fork_blockhash(ctx);
ctx->bh[i].digest[ctx->bh[i].d_len] = b64[ctx->bh[i].h % 64];
ctx->bh[i].half_digest = b64[ctx->bh[i].half_h % 64];
if(ctx->bh[i].d_len < SPAMSUM_LENGTH - 1)
{
/* We can have a problem with the tail overflowing. The
* easiest way to cope with this is to only reset the
* normal hash if we have room for more characters in
* our signature. This has the effect of combining the
* last few pieces of the message into a single piece
* */
ctx->bh[i].digest[++ctx->bh[i].d_len] = 0;
ctx->bh[i].h = HASH_INIT;
if(ctx->bh[i].d_len >= SPAMSUM_LENGTH / 2) continue;
ctx->bh[i].half_h = HASH_INIT;
ctx->bh[i].half_digest = 0;
}
else
fuzzy_try_reduce_blockhash(ctx);
}
}
AARU_LOCAL inline void roll_hash(spamsum_ctx *ctx, uint8_t c)
{
ctx->roll.h2 -= ctx->roll.h1;
ctx->roll.h2 += ROLLING_WINDOW * c;
ctx->roll.h1 += c;
ctx->roll.h1 -= ctx->roll.window[ctx->roll.n % ROLLING_WINDOW];
ctx->roll.window[ctx->roll.n % ROLLING_WINDOW] = c;
ctx->roll.n++;
/* The original spamsum AND'ed this value with 0xFFFFFFFF which
* in theory should have no effect. This AND has been removed
* for performance (jk) */
ctx->roll.h3 <<= 5;
ctx->roll.h3 ^= c;
}
AARU_LOCAL inline void fuzzy_try_reduce_blockhash(spamsum_ctx *ctx)
{
// assert(ctx->bh_start < ctx->bh_end);
if(ctx->bh_end - ctx->bh_start < 2) /* Need at least two working hashes. */
return;
if((uint64_t)SSDEEP_BS(ctx->bh_start) * SPAMSUM_LENGTH >= ctx->total_size)
/* Initial blocksize estimate would select this or a smaller
* blocksize. */
return;
if(ctx->bh[ctx->bh_start + 1].d_len < SPAMSUM_LENGTH / 2) /* Estimate adjustment would select this blocksize. */
return;
/* At this point we are clearly no longer interested in the
* start_blocksize. Get rid of it. */
++ctx->bh_start;
}
AARU_LOCAL inline void fuzzy_try_fork_blockhash(spamsum_ctx *ctx)
{
if(ctx->bh_end >= NUM_BLOCKHASHES) return;
// assert(ctx->bh_end != 0);
const uint32_t obh = ctx->bh_end - 1;
const uint32_t nbh = ctx->bh_end;
ctx->bh[nbh].h = ctx->bh[obh].h;
ctx->bh[nbh].half_h = ctx->bh[obh].half_h;
ctx->bh[nbh].digest[0] = 0;
ctx->bh[nbh].half_digest = 0;
ctx->bh[nbh].d_len = 0;
++ctx->bh_end;
}
AARU_EXPORT int AARU_CALL aaruf_spamsum_final(spamsum_ctx *ctx, uint8_t *result)
{
uint32_t bi = ctx->bh_start;
uint32_t h = ROLL_SUM(ctx);
int remain = FUZZY_MAX_RESULT - 1; /* Exclude terminating '\0'. */
if(!result) return -1;
/* Verify that our elimination was not overeager. */
// assert(bi == 0 || (uint64_t)SSDEEP_BS(bi) / 2 * SPAMSUM_LENGTH < ctx->total_size);
/* Initial blocksize guess. */
while((uint64_t)SSDEEP_BS(bi) * SPAMSUM_LENGTH < ctx->total_size)
{
++bi;
if(bi >= NUM_BLOCKHASHES)
{
errno = EOVERFLOW;
return -1;
}
}
/* Adapt blocksize guess to actual digest length. */
while(bi >= ctx->bh_end) --bi;
while(bi > ctx->bh_start && ctx->bh[bi].d_len < SPAMSUM_LENGTH / 2) --bi;
// assert(!(bi > 0 && ctx->bh[bi].d_len < SPAMSUM_LENGTH / 2));
int i = snprintf((char *)result, (size_t)remain, "%lu:", (unsigned long)SSDEEP_BS(bi));
if(i <= 0) /* Maybe snprintf has set errno here? */
return -1;
// assert(i < remain);
remain -= i;
result += i;
i = (int)ctx->bh[bi].d_len;
// assert(i <= remain);
memcpy(result, ctx->bh[bi].digest, (size_t)i);
result += i;
remain -= i;
if(h != 0)
{
// assert(remain > 0);
*result = b64[ctx->bh[bi].h % 64];
if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
{
++result;
--remain;
}
}
else if(ctx->bh[bi].digest[i] != 0)
{
// assert(remain > 0);
*result = ctx->bh[bi].digest[i];
if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
{
++result;
--remain;
}
}
// assert(remain > 0);
*result++ = ':';
--remain;
if(bi < ctx->bh_end - 1)
{
++bi;
i = (int)ctx->bh[bi].d_len;
if(i <= remain)
;
memcpy(result, ctx->bh[bi].digest, (size_t)i);
result += i;
remain -= i;
if(h != 0)
{
// assert(remain > 0);
h = ctx->bh[bi].half_h;
*result = b64[h % 64];
if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
{
++result;
--remain;
}
}
else
{
i = ctx->bh[bi].half_digest;
if(i != 0)
{
// assert(remain > 0);
*result = (uint8_t)i;
if(i < 3 || *result != result[-1] || *result != result[-2] || *result != result[-3])
{
++result;
--remain;
}
}
}
}
else if(h != 0)
{
// assert(ctx->bh[bi].d_len == 0);
// assert(remain > 0);
*result++ = b64[ctx->bh[bi].h % 64];
/* No need to bother with FUZZY_FLAG_ELIMSEQ, because this
* digest has length 1. */
--remain;
}
*result = 0;
return 0;
}
aaruformat.h
AARU_CALL
#define AARU_CALL
Definition decls.h:46
AARU_LOCAL
#define AARU_LOCAL
Definition decls.h:56
AARU_EXPORT
#define AARU_EXPORT
Definition decls.h:55
SUM_HASH
#define SUM_HASH(c, h)
Definition spamsum.c:81
b64
static uint8_t b64[]
Definition spamsum.c:32
fuzzy_try_fork_blockhash
void fuzzy_try_fork_blockhash(spamsum_ctx *ctx)
Definition spamsum.c:175
SSDEEP_BS
#define SSDEEP_BS(index)
Definition spamsum.c:82
fuzzy_engine_step
void fuzzy_engine_step(spamsum_ctx *ctx, uint8_t c)
Definition spamsum.c:84
aaruf_spamsum_free
void aaruf_spamsum_free(spamsum_ctx *ctx)
Frees a spamsum (fuzzy hash) context.
Definition spamsum.c:75
ROLL_SUM
#define ROLL_SUM(ctx)
Definition spamsum.c:80
aaruf_spamsum_init
spamsum_ctx * aaruf_spamsum_init(void)
Definition spamsum.c:37
fuzzy_try_reduce_blockhash
void fuzzy_try_reduce_blockhash(spamsum_ctx *ctx)
Definition spamsum.c:155
roll_hash
void roll_hash(spamsum_ctx *ctx, uint8_t c)
Definition spamsum.c:137
aaruf_spamsum_final
int aaruf_spamsum_final(spamsum_ctx *ctx, uint8_t *result)
Definition spamsum.c:191
aaruf_spamsum_update
int aaruf_spamsum_update(spamsum_ctx *ctx, const uint8_t *data, const uint32_t len)
Updates the spamsum context with new data.
Definition spamsum.c:59
spamsum.h
SPAMSUM_LENGTH
#define SPAMSUM_LENGTH
Definition spamsum.h:24
FUZZY_MAX_RESULT
#define FUZZY_MAX_RESULT
Definition spamsum.h:30
ROLLING_WINDOW
#define ROLLING_WINDOW
Definition spamsum.h:26
NUM_BLOCKHASHES
#define NUM_BLOCKHASHES
Definition spamsum.h:25
HASH_INIT
#define HASH_INIT
Definition spamsum.h:27
blockhash_ctx::d_len
uint32_t d_len
Definition spamsum.h:38
blockhash_ctx::h
uint32_t h
Definition spamsum.h:34
blockhash_ctx::half_digest
uint8_t half_digest
Definition spamsum.h:37
blockhash_ctx::digest
uint8_t digest[64]
Definition spamsum.h:36
blockhash_ctx::half_h
uint32_t half_h
Definition spamsum.h:35
roll_state::h2
uint32_t h2
Definition spamsum.h:45
roll_state::n
uint32_t n
Definition spamsum.h:47
roll_state::h1
uint32_t h1
Definition spamsum.h:44
roll_state::h3
uint32_t h3
Definition spamsum.h:46
roll_state::window
uint8_t window[7]
Definition spamsum.h:43
spamsum_ctx
Definition spamsum.h:51
spamsum_ctx::bh
blockhash_ctx bh[31]
Definition spamsum.h:54
spamsum_ctx::roll
roll_state roll
Definition spamsum.h:56
spamsum_ctx::bh_end
uint32_t bh_end
Definition spamsum.h:53
spamsum_ctx::total_size
uint64_t total_size
Definition spamsum.h:55
spamsum_ctx::bh_start
uint32_t bh_start
Definition spamsum.h:52