Add KenCode decompressor implementation and tests

This commit is contained in:
2026-04-20 21:57:52 +01:00
parent 16dd2ce90f
commit 5c39fda7be
11 changed files with 560 additions and 3 deletions

View File

@@ -131,7 +131,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
add_subdirectory(3rdparty)
add_library("Aaru.Compression.Native" SHARED library.c apple_rle.c apple_rle.h apple_lzh.c apple_lzh.h adc.c adc.h lzip.c flac.c flac.h
add_library("Aaru.Compression.Native" SHARED library.c apple_rle.c apple_rle.h apple_lzh.c apple_lzh.h adc.c adc.h kencode.c kencode.h lzip.c flac.c flac.h
zoo/lzd.c zoo/lzd.h zoo/lzh.c zoo/decode.c zoo/huf.c zoo/io.c zoo/lh5.c zoo/lh5.h zoo/lzh.h zoo/ar.h zoo/maketbl.c
arc/pack.c arc/squeeze.c arc/crunch.c arc/lzw.c
pak/crush.c pak/distill.c pak/bitstream.c pak/bitstream.h pak/lzw.c pak/lzw.h pak/prefixcode.c

314
kencode.c Normal file
View File

@@ -0,0 +1,314 @@
// KenCode decompressor
// Reverse-engineered from Apple DiskCopy 6.3.3 resource fork hdi2 ID 128
// Algorithm: LZSS with MSB-first bitstream and variable-length prefix codes
//
// Function map from PEF disassembly:
// 0x0000 plugin_entry Component Manager dispatcher (selectors 0-3)
// 0x00CC read_match_info Reads one token (match or literal run) from bitstream
// 0x01AC decompress Main decompression loop
// 0x03E4 bs_init Initialize bitstream reader
// 0x03F4 read_unary Count consecutive 1-bits (MSB-first)
// 0x046C read_bits Read N bits from bitstream
// 0x0510 read_literals Read N literal bytes from bitstream
// 0x05C0 decode_lit_count Decode literal run length
// 0x0750 decode_match_len Decode match length with prefix code
// 0x0A84 decode_offset Decode match offset (multi-level)
// 0x0C2C offset_dispatch Select distance class based on output position
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "library.h"
#include "kencode.h"
// ============================================================================
// MSB-first bitstream reader
// ============================================================================
typedef struct
{
const uint8_t *data;
uint32_t bit_pos;
uint32_t bit_end;
} kc_bs;
static inline void bs_init(kc_bs *bs, const uint8_t *data, size_t size)
{
bs->data = data;
bs->bit_pos = 0;
bs->bit_end = (uint32_t)(size * 8);
}
// PEF 0x046C: Read N bits MSB-first.
// The PEF reads a 32-bit big-endian word at (bit_pos / 8), shifts right
// by (32 - bit_within_byte - n), and masks to n bits.
static inline uint32_t bs_read(kc_bs *bs, int n)
{
if(n <= 0 || bs->bit_pos + (uint32_t)n > bs->bit_end) return 0;
uint32_t byte_off = bs->bit_pos >> 3;
int bit_off = bs->bit_pos & 7;
int shift = 32 - bit_off - n;
// Read a 32-bit big-endian word starting at byte_off
// Need to handle reads near end of buffer
uint32_t word = 0;
size_t max_byte = (bs->bit_end + 7) / 8;
for(int i = 0; i < 4; i++)
{
word <<= 8;
if(byte_off + i < max_byte) word |= bs->data[byte_off + i];
}
uint32_t mask = ((uint32_t)1 << n) - 1;
bs->bit_pos += n;
if(shift >= 0) return (word >> shift) & mask;
// shift < 0: need bits from next word (rare: n > 24 or misaligned)
word <<= (-shift);
if(byte_off + 4 < max_byte) word |= (bs->data[byte_off + 4] >> (8 + shift));
return word & mask;
}
// PEF 0x03F4: Count consecutive 1-bits, stopping at first 0-bit or max_count.
static inline uint32_t bs_read_unary(kc_bs *bs, int max_count)
{
uint32_t count = 0;
while((int)count < max_count && bs->bit_pos < bs->bit_end)
{
uint32_t byte_off = bs->bit_pos >> 3;
int bit_off = bs->bit_pos & 7;
bs->bit_pos++;
if(!(bs->data[byte_off] & (0x80 >> bit_off))) return count; // hit 0
count++;
}
return count;
}
// PEF 0x0510: Read N literal bytes from bitstream. Each byte is 8 bits
// at the current (possibly unaligned) bit position.
static void bs_read_bytes(kc_bs *bs, uint8_t *dst, int count)
{
uint32_t byte_off = bs->bit_pos >> 3;
int bit_off = bs->bit_pos & 7;
int rshift = 8 - bit_off;
for(int i = 0; i < count; i++)
{
uint16_t w = ((uint16_t)bs->data[byte_off] << 8) | bs->data[byte_off + 1];
dst[i] = (uint8_t)(w >> rshift);
byte_off++;
}
bs->bit_pos += count * 8;
}
// ============================================================================
// VLC decoders
// ============================================================================
// PEF 0x05C0: Decode literal run length (1..63)
static int decode_lit_count(kc_bs *bs)
{
// "0" → 1
if(bs_read(bs, 1) == 0) return 1;
// "1" + 2 bits
uint32_t v = bs_read(bs, 2);
if(v == 0) return 2;
if(v == 1) return 3;
if(v == 2) return (int)bs_read(bs, 2) + 4; // 4-7
// v==3: 4 more bits
uint32_t e = bs_read(bs, 4);
if(e <= 7) return (int)e + 8; // 8-15
if(e <= 11) return (int)(e * 4 + bs_read(bs, 2)) - 16; // 16-31
return (int)(e * 8 + bs_read(bs, 3)) - 64; // 32-63
}
// PEF 0x0750: Decode match length raw value (0..2042+)
// Value 0 with had_literals=1 → literal run marker
// Otherwise: actual match length = raw + 2 (or +3 if previous was literals)
static int decode_match_len(kc_bs *bs)
{
uint32_t pfx = bs_read_unary(bs, 10);
switch(pfx)
{
case 0:
return (int)bs_read(bs, 1); // 0-1
case 1:
{
uint32_t b = bs_read(bs, 1);
if(b == 0) return 2;
return (int)bs_read(bs, 1) + 3; // 3-4
}
case 2:
{
uint32_t b = bs_read(bs, 1);
if(b == 0) return (int)bs_read(bs, 1) + 5; // 5-6
return (int)bs_read(bs, 2) + 7; // 7-10
}
case 3:
return (int)bs_read(bs, 3) + 11; // 11-18
case 4:
return (int)bs_read(bs, 3) + 19; // 19-26
case 5:
return (int)bs_read(bs, 5) + 27; // 27-58
case 6:
return (int)bs_read(bs, 6) + 59; // 59-122
case 7:
return (int)bs_read(bs, 7) + 123; // 123-250
case 8:
return (int)bs_read(bs, 8) + 251; // 251-506
case 9:
return (int)bs_read(bs, 9) + 507; // 507-1018
default:
return (int)bs_read(bs, 10) + 1019; // 1019+
}
}
// PEF 0x0A84: Decode match offset given distance class (dist_bits)
static int decode_offset(kc_bs *bs, int dist_bits, int hash_size)
{
int pw = 1 << dist_bits;
// Prefix "0" + dist_bits → short offset
if(bs_read(bs, 1) == 0) return (int)bs_read(bs, dist_bits) + 1;
// Prefix "10" + (dist_bits+2) bits → medium offset
if(bs_read(bs, 1) == 0) return pw + 1 + (int)bs_read(bs, dist_bits + 2);
// Prefix "11" → long offset, escalating levels
int pw5 = pw * 5;
if(hash_size <= pw5 + 2) return pw5 + 1 + (int)bs_read(bs, 1);
if(hash_size <= pw5 + 4) return pw5 + 1 + (int)bs_read(bs, 2);
// Escalating loop
int max_level = dist_bits + 4;
int scale = 4;
int level = 3;
int threshold = pw5 + 4;
if(max_level >= 3)
{
while(level <= max_level)
{
threshold += scale;
// 68K helper at 0x0ABA: clamp ONLY the exact value 0x680 to 0x66C
int clamped = (threshold == 0x680) ? 0x66C : threshold;
if(hash_size <= clamped || level == max_level) break;
scale <<= 1;
level++;
}
}
return pw5 + 1 + (int)bs_read(bs, level);
}
// PEF 0x0C2C / 68K 0x0D48: Select distance class based on output position and call decode_offset
// out_pos determines the distance class, hash_size (from state init) caps it for large positions
static int offset_dispatch(kc_bs *bs, int out_pos, int hash_size)
{
int dist_bits;
if(out_pos <= 0xa)
dist_bits = 0;
else if(out_pos <= 0x14)
dist_bits = 1;
else if(out_pos <= 0x28)
dist_bits = 2;
else if(out_pos <= 0x50)
dist_bits = 3;
else if(out_pos <= 0xa0)
dist_bits = 4;
else if(out_pos <= 0x2a0)
dist_bits = 5;
else if(out_pos <= 0x3e8)
dist_bits = 6;
else if(out_pos <= 0xa80 || hash_size <= 0x800)
dist_bits = 7;
else if(out_pos <= 0x1500 || hash_size <= 0x1000)
dist_bits = 8;
else if(out_pos <= 0x2a00 || hash_size <= 0x2000)
dist_bits = 9;
else if(out_pos <= 0x5400 || hash_size <= 0x4000)
dist_bits = 10;
else if(out_pos <= 0xa800 || hash_size <= 0x8000)
dist_bits = 11;
else if(out_pos <= 0x11170 || hash_size <= 0x10000)
dist_bits = 12;
else if(out_pos <= 0x2a000 || hash_size <= 0x20000)
dist_bits = 13;
else
dist_bits = 14;
// 68K passes (out_pos, dist_bits, bitstream) to decode_offset
// but decode_offset internally uses out_pos for hash_size comparisons too
return decode_offset(bs, dist_bits, out_pos);
}
// ============================================================================
// Main decompressor — PEF 0x01AC
// ============================================================================
AARU_EXPORT int32_t AARU_CALL AARU_kencode_decode_buffer(uint8_t *dst_buffer, size_t *dst_size,
const uint8_t *src_buffer, size_t src_size)
{
if(!dst_buffer || !dst_size || !src_buffer || src_size == 0) return -1;
size_t max_out = *dst_size;
kc_bs bs;
bs_init(&bs, src_buffer, src_size);
size_t out_pos = 0;
uint8_t had_literals = 1; // PEF 0x01C0-0x01C4
// Plugin init: hash_size from clamp_hash_size(0x2800) → 0x2800 (in range [0x800, 0x30000])
int hash_size = 0x2800;
while(out_pos < max_out)
{
// PEF 0x00CC: read_match_info
int raw = decode_match_len(&bs);
if(raw <= 0 && had_literals)
{
// Literal run
int lit_count = decode_lit_count(&bs);
if(out_pos + lit_count > max_out) return -4;
bs_read_bytes(&bs, dst_buffer + out_pos, lit_count);
out_pos += lit_count;
// 68K 0x03C4-0x03D2: had_literals = (lit_count >= 63) ? 1 : 0
// PPC 0x0180-0x0194: equivalent via subfc/adde/subfe/clrlwi
had_literals = (lit_count >= 63) ? 1 : 0;
}
else
{
// Match reference
int match_len;
if(had_literals == 0)
match_len = raw + 3; // PEF 0x0130: +3 when coming after non-literal
else
match_len = raw + 2; // PEF 0x011C: +2 when had_literals is set
had_literals = 1; // PEF 0x0140
int offset = offset_dispatch(&bs, (int)out_pos, hash_size);
if(out_pos + match_len > max_out) return -4;
if(offset > (int)out_pos) return -4;
// Byte-by-byte copy (may overlap)
for(int i = 0; i < match_len; i++) dst_buffer[out_pos + i] = dst_buffer[out_pos - offset + i];
out_pos += match_len;
}
}
*dst_size = out_pos;
return 0;
}

12
kencode.h Normal file
View File

@@ -0,0 +1,12 @@
// KenCode decompressor
// Reverse-engineered from Apple DiskCopy 6.3.3 hdi2 resource ID 128
#ifndef KENCODE_H
#define KENCODE_H
#include <stddef.h>
#include <stdint.h>
int kencode_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, size_t src_size);
#endif // KENCODE_H

View File

@@ -57,6 +57,10 @@
AARU_EXPORT int32_t AARU_CALL AARU_adc_decode_buffer(uint8_t *dst_buffer, int32_t dst_size, const uint8_t *src_buffer,
int32_t src_size);
// KenCode decompression (NDIF chunk type 0x80)
AARU_EXPORT int32_t AARU_CALL AARU_kencode_decode_buffer(uint8_t *dst_buffer, size_t *dst_size,
const uint8_t *src_buffer, size_t src_size);
AARU_EXPORT int32_t AARU_CALL AARU_apple_rle_decode_buffer(uint8_t *dst_buffer, int32_t dst_size,
const uint8_t *src_buffer, int32_t src_size);
@@ -402,7 +406,7 @@ AARU_EXPORT int AARU_CALL AARU_stuffit_arsenic_decode_buffer(uint8_t *dst_buffer
// ShrinkWrap 3 SIT2 compression (NDIF chunk type 0xF0)
AARU_EXPORT int AARU_CALL AARU_stuffit_shrinkwrap_decode_buffer(uint8_t *dst_buffer, size_t *dst_size,
const uint8_t *src_buffer, size_t src_size);
const uint8_t *src_buffer, size_t src_size);
// StuffIt X method 0: Brimstone (PPMd Variant G)
AARU_EXPORT int AARU_CALL AARU_stuffitx_brimstone_decode_buffer(uint8_t *dst_buffer, size_t *dst_size,

View File

@@ -222,6 +222,21 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/mf1dd_mfs_fast.dart.lz
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/mf1dd_mfs_best.dart.lz
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/kencode_boot.bin
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/kencode_zeros.bin
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/kencode_hfs.bin
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/kencode_last.bin
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/kencode_large.bin
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
# 'Google_Tests_run' is the target name
# 'test1.cpp tests2.cpp' are source files with tests
add_executable(tests_run apple_rle.cpp crc32.c crc32.h adc.cpp bzip2.cpp lzip.cpp lzfse.cpp zstd.cpp lzma.cpp flac.cpp lz4.cpp
@@ -236,5 +251,6 @@ add_executable(tests_run apple_rle.cpp crc32.c crc32.h adc.cpp bzip2.cpp lzip.cp
cpt/cpt.cpp
dd/dd.cpp
stuffit/stuffit.cpp
dart/dart.cpp)
dart/dart.cpp
kencode.cpp)
target_link_libraries(tests_run gtest gtest_main "Aaru.Compression.Native")

BIN
tests/data/kencode_boot.bin Normal file

Binary file not shown.

BIN
tests/data/kencode_hfs.bin Normal file

Binary file not shown.

Binary file not shown.

BIN
tests/data/kencode_last.bin Normal file

Binary file not shown.

Binary file not shown.

211
tests/kencode.cpp Normal file
View File

@@ -0,0 +1,211 @@
/*
* This file is part of the Aaru Data Preservation Suite.
* Copyright (c) 2019-2026 Natalia Portillo.
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include <unistd.h>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include "../library.h"
#include "../kencode.h"
#include "crc32.h"
#include "gtest/gtest.h"
// DOS_720.img chunk 0: boot sector with mixed data/matches (2495 → 10240)
#define KC_BOOT_COMP_SIZE 2495
#define KC_BOOT_DECOMP_SIZE 10240
#define KC_BOOT_CRC32 0x18DDE60Cu
// DOS_720.img chunk 47: mostly zero data, high compression (442 → 10240)
#define KC_ZEROS_COMP_SIZE 442
#define KC_ZEROS_DECOMP_SIZE 10240
#define KC_ZEROS_CRC32 0x271DDE9Au
// HFS_800.img chunk 2: HFS filesystem data (4579 → 10240)
#define KC_HFS_COMP_SIZE 4579
#define KC_HFS_DECOMP_SIZE 10240
#define KC_HFS_CRC32 0x72B8E8F1u
// HFS_DMF.img chunk 77: last chunk, non-standard 6144 byte output (3076 → 6144)
#define KC_LAST_COMP_SIZE 3076
#define KC_LAST_DECOMP_SIZE 6144
#define KC_LAST_CRC32 0xA0B2EAEFu
// DOS_SP_5Mb.img chunk 0: 5MB disk image first chunk (2017 → 10240)
#define KC_LARGE_COMP_SIZE 2017
#define KC_LARGE_DECOMP_SIZE 10240
#define KC_LARGE_CRC32 0xA584FF1Au
static uint8_t *read_test_file(const char *name, size_t expected_size)
{
char path[PATH_MAX];
char filename[PATH_MAX];
getcwd(path, PATH_MAX);
snprintf(filename, PATH_MAX, "%s/data/%s", path, name);
FILE *file = fopen(filename, "rb");
if(file == nullptr) return nullptr;
auto *buf = (uint8_t *)malloc(expected_size);
fread(buf, 1, expected_size, file);
fclose(file);
return buf;
}
/* ---- Boot sector: mixed data and matches ---- */
static const uint8_t *kc_boot_buffer;
class KencodeBootFixture : public ::testing::Test
{
protected:
void SetUp() { kc_boot_buffer = read_test_file("kencode_boot.bin", KC_BOOT_COMP_SIZE); }
void TearDown() { free((void *)kc_boot_buffer); }
};
TEST_F(KencodeBootFixture, kencode_boot)
{
auto *outBuf = (uint8_t *)malloc(KC_BOOT_DECOMP_SIZE);
size_t destLen = KC_BOOT_DECOMP_SIZE;
auto err = AARU_kencode_decode_buffer(outBuf, &destLen, kc_boot_buffer, KC_BOOT_COMP_SIZE);
EXPECT_EQ(err, 0);
EXPECT_EQ(destLen, (size_t)KC_BOOT_DECOMP_SIZE);
auto crc = crc32_data(outBuf, KC_BOOT_DECOMP_SIZE);
free(outBuf);
EXPECT_EQ(crc, KC_BOOT_CRC32);
}
/* ---- Mostly zeros: high compression ratio (23:1) ---- */
static const uint8_t *kc_zeros_buffer;
class KencodeZerosFixture : public ::testing::Test
{
protected:
void SetUp() { kc_zeros_buffer = read_test_file("kencode_zeros.bin", KC_ZEROS_COMP_SIZE); }
void TearDown() { free((void *)kc_zeros_buffer); }
};
TEST_F(KencodeZerosFixture, kencode_zeros)
{
auto *outBuf = (uint8_t *)malloc(KC_ZEROS_DECOMP_SIZE);
size_t destLen = KC_ZEROS_DECOMP_SIZE;
auto err = AARU_kencode_decode_buffer(outBuf, &destLen, kc_zeros_buffer, KC_ZEROS_COMP_SIZE);
EXPECT_EQ(err, 0);
EXPECT_EQ(destLen, (size_t)KC_ZEROS_DECOMP_SIZE);
auto crc = crc32_data(outBuf, KC_ZEROS_DECOMP_SIZE);
free(outBuf);
EXPECT_EQ(crc, KC_ZEROS_CRC32);
}
/* ---- HFS filesystem data ---- */
static const uint8_t *kc_hfs_buffer;
class KencodeHfsFixture : public ::testing::Test
{
protected:
void SetUp() { kc_hfs_buffer = read_test_file("kencode_hfs.bin", KC_HFS_COMP_SIZE); }
void TearDown() { free((void *)kc_hfs_buffer); }
};
TEST_F(KencodeHfsFixture, kencode_hfs)
{
auto *outBuf = (uint8_t *)malloc(KC_HFS_DECOMP_SIZE);
size_t destLen = KC_HFS_DECOMP_SIZE;
auto err = AARU_kencode_decode_buffer(outBuf, &destLen, kc_hfs_buffer, KC_HFS_COMP_SIZE);
EXPECT_EQ(err, 0);
EXPECT_EQ(destLen, (size_t)KC_HFS_DECOMP_SIZE);
auto crc = crc32_data(outBuf, KC_HFS_DECOMP_SIZE);
free(outBuf);
EXPECT_EQ(crc, KC_HFS_CRC32);
}
/* ---- Last chunk: non-standard 6144-byte output ---- */
static const uint8_t *kc_last_buffer;
class KencodeLastFixture : public ::testing::Test
{
protected:
void SetUp() { kc_last_buffer = read_test_file("kencode_last.bin", KC_LAST_COMP_SIZE); }
void TearDown() { free((void *)kc_last_buffer); }
};
TEST_F(KencodeLastFixture, kencode_last)
{
auto *outBuf = (uint8_t *)malloc(KC_LAST_DECOMP_SIZE);
size_t destLen = KC_LAST_DECOMP_SIZE;
auto err = AARU_kencode_decode_buffer(outBuf, &destLen, kc_last_buffer, KC_LAST_COMP_SIZE);
EXPECT_EQ(err, 0);
EXPECT_EQ(destLen, (size_t)KC_LAST_DECOMP_SIZE);
auto crc = crc32_data(outBuf, KC_LAST_DECOMP_SIZE);
free(outBuf);
EXPECT_EQ(crc, KC_LAST_CRC32);
}
/* ---- 5MB disk image chunk ---- */
static const uint8_t *kc_large_buffer;
class KencodeLargeFixture : public ::testing::Test
{
protected:
void SetUp() { kc_large_buffer = read_test_file("kencode_large.bin", KC_LARGE_COMP_SIZE); }
void TearDown() { free((void *)kc_large_buffer); }
};
TEST_F(KencodeLargeFixture, kencode_large)
{
auto *outBuf = (uint8_t *)malloc(KC_LARGE_DECOMP_SIZE);
size_t destLen = KC_LARGE_DECOMP_SIZE;
auto err = AARU_kencode_decode_buffer(outBuf, &destLen, kc_large_buffer, KC_LARGE_COMP_SIZE);
EXPECT_EQ(err, 0);
EXPECT_EQ(destLen, (size_t)KC_LARGE_DECOMP_SIZE);
auto crc = crc32_data(outBuf, KC_LARGE_DECOMP_SIZE);
free(outBuf);
EXPECT_EQ(crc, KC_LARGE_CRC32);
}