diff --git a/CMakeLists.txt b/CMakeLists.txt index 03c3461..4519c2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,7 +176,48 @@ add_library("Aaru.Compression.Native" SHARED library.c apple_rle.c apple_rle.h a arj/arj.h arj/arj_fastest.c arjz/arjz.c - arjz/arjz.h) + arjz/arjz.h + zip/shrink.c + zip/shrink.h + zip/reduce.c + zip/reduce.h + zip/implode.c + zip/implode.h + zip/deflate64.c + zip/deflate64.h + zip/zip.c + zip/zip.h + ppmd/RangeCoder.c + ppmd/RangeCoder.h + ppmd/Context.c + ppmd/Context.h + ppmd/SubAllocator.h + ppmd/SubAllocatorVariantI.c + ppmd/SubAllocatorVariantI.h + ppmd/VariantI.c + ppmd/VariantI.h + wavpack/common_utils.c + wavpack/decorr_utils.c + wavpack/entropy_utils.c + wavpack/open_legacy.c + wavpack/open_utils.c + wavpack/read_words.c + wavpack/tags.c + wavpack/unpack.c + wavpack/unpack_floats.c + wavpack/unpack_seek.c + wavpack/unpack_utils.c + wavpack/wavpack.h + wavpack/wavpack_local.h + wavpack/wavpack_version.h + winzipjpeg/ArithmeticDecoder.c + winzipjpeg/ArithmeticDecoder.h + winzipjpeg/Decompressor.c + winzipjpeg/Decompressor.h + winzipjpeg/JPEG.c + winzipjpeg/JPEG.h + winzipjpeg/InputStream.h + winzipjpeg/LZMA.h) include(3rdparty/bzip2.cmake) include(3rdparty/flac.cmake) diff --git a/library.c b/library.c index 41a5ef6..04eb107 100644 --- a/library.c +++ b/library.c @@ -29,6 +29,8 @@ #include "3rdparty/lzfse/src/lzvn_encode_base.h" #include "3rdparty/lzma/C/7zCrc.h" #include "3rdparty/lzma/C/Alloc.h" +#include "3rdparty/lzma/C/Lzma2Dec.h" +#include "3rdparty/lzma/C/Lzma2Enc.h" #include "3rdparty/lzma/C/LzmaLib.h" #include "3rdparty/lzma/C/Xz.h" #include "3rdparty/lzma/C/XzCrc64.h" @@ -46,6 +48,7 @@ #include "3rdparty/lzo-2.10/include/lzo/lzodefs.h" #include "3rdparty/zstd/lib/zstd.h" #include "ace/ace.h" +#include "zip/zip.h" AARU_EXPORT int32_t AARU_CALL AARU_bzip2_decode_buffer(uint8_t *dst_buffer, uint32_t *dst_size, const uint8_t *src_buffer, uint32_t src_size) @@ -470,4 +473,86 @@ AARU_EXPORT int AARU_CALL ace_decompress_blocked(const uint8_t *in_buf, size_t i return ret; } +/* ============== LZMA2 ============== */ + +AARU_EXPORT int32_t AARU_CALL AARU_lzma2_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, + size_t *src_size, uint8_t prop) +{ + ELzmaStatus status; + return Lzma2Decode(dst_buffer, (SizeT *)dst_size, src_buffer, (SizeT *)src_size, prop, LZMA_FINISH_END, &status, + &g_Alloc); +} + +AARU_EXPORT int32_t AARU_CALL AARU_lzma2_encode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, + size_t src_size, uint8_t *outProp, int32_t level, + uint32_t dictSize, int32_t lc, int32_t lp, int32_t pb, + int32_t fb, int32_t numThreads) +{ + CLzma2EncHandle enc; + CLzma2EncProps props; + SRes res; + + enc = Lzma2Enc_Create(&g_Alloc, &g_Alloc); + if(!enc) return SZ_ERROR_MEM; + + Lzma2EncProps_Init(&props); + props.lzmaProps.level = level; + props.lzmaProps.dictSize = dictSize; + props.lzmaProps.lc = lc; + props.lzmaProps.lp = lp; + props.lzmaProps.pb = pb; + props.lzmaProps.fb = fb; + props.lzmaProps.numThreads = numThreads; + + res = Lzma2Enc_SetProps(enc, &props); + if(res != SZ_OK) + { + Lzma2Enc_Destroy(enc); + return res; + } + + *outProp = Lzma2Enc_WriteProperties(enc); + + res = Lzma2Enc_Encode2(enc, NULL, dst_buffer, dst_size, NULL, src_buffer, src_size, NULL); + + Lzma2Enc_Destroy(enc); + return res; +} + +/* ============== ZIP Wrappers ============== */ + +AARU_EXPORT int AARU_CALL AARU_zip_shrink_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size) +{ return zip_shrink_decompress(src_buffer, src_size, dst_buffer, dst_size); } + +AARU_EXPORT int AARU_CALL AARU_zip_reduce_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, int comp_factor) +{ return zip_reduce_decompress(src_buffer, src_size, dst_buffer, dst_size, comp_factor); } + +AARU_EXPORT int AARU_CALL AARU_zip_implode_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, + int large_dictionary, int has_literals) +{ return zip_implode_decompress(src_buffer, src_size, dst_buffer, dst_size, large_dictionary, has_literals); } + +AARU_EXPORT int AARU_CALL AARU_zip_deflate64_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size) +{ return zip_deflate64_decompress(src_buffer, src_size, dst_buffer, dst_size); } + +AARU_EXPORT int AARU_CALL AARU_zip_ppmd_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, + size_t src_size, int max_order, int sub_alloc_size, + int restoration) +{ return zip_ppmd_decompress(dst_buffer, dst_size, src_buffer, src_size, max_order, sub_alloc_size, restoration); } + +AARU_EXPORT int AARU_CALL AARU_zip_wavpack_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, + uint32_t num_samples, int bits_per_sample, int num_channels) +{ + return zip_wavpack_decompress(dst_buffer, dst_size, src_buffer, src_size, num_samples, bits_per_sample, + num_channels); +} + +AARU_EXPORT int AARU_CALL AARU_zip_winzipjpeg_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size) +{ return zip_winzipjpeg_decompress(dst_buffer, dst_size, src_buffer, src_size); } + AARU_EXPORT uint64_t AARU_CALL AARU_get_acn_version() { return AARU_CHECKUMS_NATIVE_VERSION; } \ No newline at end of file diff --git a/library.h b/library.h index 3cd7d57..dc9b549 100644 --- a/library.h +++ b/library.h @@ -71,10 +71,10 @@ AARU_EXPORT size_t AARU_CALL AARU_flac_encode_redbook_buffer( uint32_t application_id_len); AARU_EXPORT int32_t AARU_CALL AARU_lz4_decode_buffer(uint8_t *dst_buffer, int32_t dst_size, const uint8_t *src_buffer, - int32_t src_size); + int32_t src_size); AARU_EXPORT int32_t AARU_CALL AARU_lz4_encode_buffer(uint8_t *dst_buffer, int32_t dst_size, const uint8_t *src_buffer, - int32_t src_size); + int32_t src_size); AARU_EXPORT int32_t AARU_CALL AARU_lzip_decode_buffer(uint8_t *dst_buffer, int32_t dst_size, const uint8_t *src_buffer, int32_t src_size); @@ -125,16 +125,17 @@ AARU_EXPORT size_t AARU_CALL AARU_zstd_encode_buffer(void *dst_buffer, size_t ds /** * LZO Algorithm Types */ -typedef enum { - AARU_LZO_ALGORITHM_LZO1 = 0, /* LZO1 algorithm */ - AARU_LZO_ALGORITHM_LZO1A = 1, /* LZO1A algorithm */ - AARU_LZO_ALGORITHM_LZO1B = 2, /* LZO1B algorithm (supports compression levels 1-9, 99, 999) */ - AARU_LZO_ALGORITHM_LZO1C = 3, /* LZO1C algorithm (supports compression levels 1-9, 99, 999) */ - AARU_LZO_ALGORITHM_LZO1F = 4, /* LZO1F algorithm (supports compression level 999) */ - AARU_LZO_ALGORITHM_LZO1X = 5, /* LZO1X algorithm (supports compression levels 11, 12, 15, 999) - most common */ - AARU_LZO_ALGORITHM_LZO1Y = 6, /* LZO1Y algorithm (supports compression level 999) */ - AARU_LZO_ALGORITHM_LZO1Z = 7, /* LZO1Z algorithm (only 999 compression level) */ - AARU_LZO_ALGORITHM_LZO2A = 8 /* LZO2A algorithm (only 999 compression level) */ +typedef enum +{ + AARU_LZO_ALGORITHM_LZO1 = 0, /* LZO1 algorithm */ + AARU_LZO_ALGORITHM_LZO1A = 1, /* LZO1A algorithm */ + AARU_LZO_ALGORITHM_LZO1B = 2, /* LZO1B algorithm (supports compression levels 1-9, 99, 999) */ + AARU_LZO_ALGORITHM_LZO1C = 3, /* LZO1C algorithm (supports compression levels 1-9, 99, 999) */ + AARU_LZO_ALGORITHM_LZO1F = 4, /* LZO1F algorithm (supports compression level 999) */ + AARU_LZO_ALGORITHM_LZO1X = 5, /* LZO1X algorithm (supports compression levels 11, 12, 15, 999) - most common */ + AARU_LZO_ALGORITHM_LZO1Y = 6, /* LZO1Y algorithm (supports compression level 999) */ + AARU_LZO_ALGORITHM_LZO1Z = 7, /* LZO1Z algorithm (only 999 compression level) */ + AARU_LZO_ALGORITHM_LZO2A = 8 /* LZO2A algorithm (only 999 compression level) */ } aaru_lzo_algorithm_t; AARU_EXPORT int32_t AARU_CALL AARU_lzo_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, @@ -192,14 +193,17 @@ AARU_EXPORT int AARU_CALL pak_decompress_distill(const unsigned char *in_buf, si /** * HA Algorithm Types */ -typedef enum { - HA_ALGORITHM_ASC = 0, /* ASC algorithm */ - HA_ALGORITHM_HSC = 1 /* HSC algorithm */ +typedef enum +{ + HA_ALGORITHM_ASC = 0, /* ASC algorithm */ + HA_ALGORITHM_HSC = 1 /* HSC algorithm */ } ha_algorithm_t; -AARU_EXPORT int AARU_CALL ha_asc_decompress(const unsigned char *in_buf, size_t in_len, unsigned char *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL ha_asc_decompress(const unsigned char *in_buf, size_t in_len, unsigned char *out_buf, + size_t *out_len); -AARU_EXPORT int AARU_CALL ha_hsc_decompress(const unsigned char *in_buf, size_t in_len, unsigned char *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL ha_hsc_decompress(const unsigned char *in_buf, size_t in_len, unsigned char *out_buf, + size_t *out_len); // LHA -lh1- (Dynamic Huffman, 4KB window) AARU_EXPORT int AARU_CALL lha_decompress_lh1(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); @@ -235,34 +239,84 @@ AARU_EXPORT int AARU_CALL pmarc_decompress_pm1(const uint8_t *in_buf, size_t in_ AARU_EXPORT int AARU_CALL pmarc_decompress_pm2(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); // ACE v1 (LZ77) decompression -AARU_EXPORT int AARU_CALL ace_decompress_lz77(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, int dic_bits); +AARU_EXPORT int AARU_CALL ace_decompress_lz77(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, + int dic_bits); // ACE v2 (Blocked) decompression -AARU_EXPORT int AARU_CALL ace_decompress_blocked(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, int dic_bits); +AARU_EXPORT int AARU_CALL ace_decompress_blocked(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len, int dic_bits); // ARJ Method 1 (LZH, most compression) -AARU_EXPORT int AARU_CALL arj_decompress_method1(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arj_decompress_method1(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJ Method 2 (LZH, medium compression) -AARU_EXPORT int AARU_CALL arj_decompress_method2(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arj_decompress_method2(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJ Method 3 (LZH, fast compression) -AARU_EXPORT int AARU_CALL arj_decompress_method3(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arj_decompress_method3(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJ Method 4 (Fastest, variable-width LZSS) -AARU_EXPORT int AARU_CALL arj_decompress_fastest(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arj_decompress_fastest(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJZ Method 1 (LZH, 64KB window) -AARU_EXPORT int AARU_CALL arjz_decompress_method1(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arjz_decompress_method1(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJZ Method 2 (LZH, 64KB window) -AARU_EXPORT int AARU_CALL arjz_decompress_method2(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arjz_decompress_method2(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJZ Method 3 (LZH, 64KB window) -AARU_EXPORT int AARU_CALL arjz_decompress_method3(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); +AARU_EXPORT int AARU_CALL arjz_decompress_method3(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len); // ARJZ custom extended DEFLATE decompression -AARU_EXPORT int AARU_CALL arjz_decompress_buffer(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, - size_t orig_size); +AARU_EXPORT int AARU_CALL arjz_decompress_buffer(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, + size_t *out_len, size_t orig_size); + +// LZMA2 decode (single prop byte instead of 5-byte props blob) +AARU_EXPORT int32_t AARU_CALL AARU_lzma2_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, + size_t *src_size, uint8_t prop); + +// LZMA2 encode +AARU_EXPORT int32_t AARU_CALL AARU_lzma2_encode_buffer(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, + size_t src_size, uint8_t *outProp, int32_t level, + uint32_t dictSize, int32_t lc, int32_t lp, int32_t pb, + int32_t fb, int32_t numThreads); + +// ZIP method 1: Shrink (LZW, 9-13 bit codes) +AARU_EXPORT int AARU_CALL AARU_zip_shrink_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size); + +// ZIP methods 2-5: Reduce (follower sets + LZ77, compression factor 1-4) +AARU_EXPORT int AARU_CALL AARU_zip_reduce_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, int comp_factor); + +// ZIP method 6: Implode (Shannon-Fano + LZSS) +AARU_EXPORT int AARU_CALL AARU_zip_implode_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, + int large_dictionary, int has_literals); + +// ZIP method 9: Deflate64 +AARU_EXPORT int AARU_CALL AARU_zip_deflate64_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size); + +// ZIP method 98: PPMd variant I +AARU_EXPORT int AARU_CALL AARU_zip_ppmd_decode_buffer(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, + size_t src_size, int max_order, int sub_alloc_size, + int restoration); + +// ZIP method 97: WinZip WavPack +AARU_EXPORT int AARU_CALL AARU_zip_wavpack_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size, + uint32_t num_samples, int bits_per_sample, int num_channels); + +// ZIP method 96: WinZip JPEG +AARU_EXPORT int AARU_CALL AARU_zip_winzipjpeg_decode_buffer(uint8_t *dst_buffer, size_t *dst_size, + const uint8_t *src_buffer, size_t src_size); #endif // AARU_COMPRESSION_NATIVE_LIBRARY_H diff --git a/ppmd/Context.c b/ppmd/Context.c new file mode 100644 index 0000000..0fe4564 --- /dev/null +++ b/ppmd/Context.c @@ -0,0 +1,341 @@ +/* + * Context.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "Context.h" + +#include + +SEE2Context MakeSEE2(int initval, int count) +{ + SEE2Context self; + self.Shift = PERIOD_BITS - 4; + self.Summ = initval << self.Shift; + self.Count = count; + return self; +} + +unsigned int GetSEE2MeanMasked(SEE2Context *self) +{ + unsigned int retval = self->Summ >> self->Shift; + self->Summ -= retval; + retval &= 0x03ff; + if(retval == 0) return 1; + return retval; +} + +unsigned int GetSEE2Mean(SEE2Context *self) +{ + unsigned int retval = self->Summ >> self->Shift; + self->Summ -= retval; + if(retval == 0) return 1; + return retval; +} + +void UpdateSEE2(SEE2Context *self) +{ + if(self->Shift >= PERIOD_BITS) return; + + self->Count--; + if(self->Count == 0) + { + self->Summ *= 2; + self->Count = 3 << self->Shift; + self->Shift++; + } +} + +PPMdContext *PPMdStateSuccessor(PPMdState *self, PPMdCoreModel *model) +{ return OffsetToPointer(model->alloc, self->Successor); } + +void SetPPMdStateSuccessorPointer(PPMdState *self, PPMdContext *newsuccessor, PPMdCoreModel *model) +{ self->Successor = PointerToOffset(model->alloc, newsuccessor); } + +PPMdState *PPMdContextStates(PPMdContext *self, PPMdCoreModel *model) +{ return OffsetToPointer(model->alloc, self->States); } + +void SetPPMdContextStatesPointer(PPMdContext *self, PPMdState *newstates, PPMdCoreModel *model) +{ self->States = PointerToOffset(model->alloc, newstates); } + +PPMdContext *PPMdContextSuffix(PPMdContext *self, PPMdCoreModel *model) +{ return OffsetToPointer(model->alloc, self->Suffix); } + +void SetPPMdContextSuffixPointer(PPMdContext *self, PPMdContext *newsuffix, PPMdCoreModel *model) +{ self->Suffix = PointerToOffset(model->alloc, newsuffix); } + +PPMdState *PPMdContextOneState(PPMdContext *self) { return (PPMdState *)&self->SummFreq; } + +PPMdContext *NewPPMdContext(PPMdCoreModel *model) +{ + PPMdContext *context = OffsetToPointer(model->alloc, AllocContext(model->alloc)); + if(context) + { + context->LastStateIndex = 0; + context->Flags = 0; + context->Suffix = 0; + } + return context; +} + +PPMdContext *NewPPMdContextAsChildOf(PPMdCoreModel *model, PPMdContext *suffixcontext, PPMdState *suffixstate, + PPMdState *firststate) +{ + PPMdContext *context = OffsetToPointer(model->alloc, AllocContext(model->alloc)); + if(context) + { + context->LastStateIndex = 0; + context->Flags = 0; + SetPPMdContextSuffixPointer(context, suffixcontext, model); + SetPPMdStateSuccessorPointer(suffixstate, context, model); + if(firststate) *(PPMdContextOneState(context)) = *firststate; + } + return context; +} + +// Tabulated escapes for exponential symbol distribution +static const uint8_t ExpEscape[16] = {25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2}; + +#define GET_MEAN(SUMM, SHIFT, ROUND) ((SUMM + (1 << (SHIFT - ROUND))) >> (SHIFT)) + +void PPMdDecodeBinSymbol(PPMdContext *self, PPMdCoreModel *model, uint16_t *bs, int freqlimit, bool altnextbit) +{ + PPMdState *rs = PPMdContextOneState(self); + + int bit; + if(altnextbit) + bit = NextWeightedBitFromPPMdRangeCoder2(&model->coder, *bs, TOT_BITS); + else + bit = NextWeightedBitFromPPMdRangeCoder(&model->coder, *bs, 1 << TOT_BITS); + + if(bit == 0) + { + model->PrevSuccess = 1; + model->RunLength++; + model->FoundState = rs; + + if(rs->Freq < freqlimit) rs->Freq++; + *bs += INTERVAL - GET_MEAN(*bs, PERIOD_BITS, 2); + } + else + { + model->PrevSuccess = 0; + model->FoundState = NULL; + model->LastMaskIndex = 0; + model->CharMask[rs->Symbol] = model->EscCount; + + *bs -= GET_MEAN(*bs, PERIOD_BITS, 2); + model->InitEsc = ExpEscape[*bs >> 10]; + } +} + +int PPMdDecodeSymbol1(PPMdContext *self, PPMdCoreModel *model, bool greaterorequal) +{ + model->scale = self->SummFreq; + + PPMdState *states = PPMdContextStates(self, model); + int firstcount = states[0].Freq; + int count = PPMdRangeCoderCurrentCount(&model->coder, model->scale); + int adder = greaterorequal ? 1 : 0; + + if(count < firstcount) + { + RemovePPMdRangeCoderSubRange(&model->coder, 0, firstcount); + if(2 * firstcount + adder > model->scale) + { + model->PrevSuccess = 1; + model->RunLength++; + } + else + model->PrevSuccess = 0; + + model->FoundState = &states[0]; + states[0].Freq = firstcount + 4; + self->SummFreq += 4; + + if(firstcount + 4 > MAX_FREQ) model->RescalePPMdContext(self, model); + + return -1; + } + + int highcount = firstcount; + model->PrevSuccess = 0; + + for(int i = 1; i <= self->LastStateIndex; i++) + { + highcount += states[i].Freq; + if(highcount > count) + { + RemovePPMdRangeCoderSubRange(&model->coder, highcount - states[i].Freq, highcount); + UpdatePPMdContext1(self, model, &states[i]); + return -1; + } + } + + if(!model->FoundState) return -1; + int lastsym = model->FoundState->Symbol; + + // if ( Suffix ) PrefetchData(Suffix); + RemovePPMdRangeCoderSubRange(&model->coder, highcount, model->scale); + model->LastMaskIndex = self->LastStateIndex; + model->FoundState = NULL; + + for(int i = 0; i <= self->LastStateIndex; i++) model->CharMask[states[i].Symbol] = model->EscCount; + + return lastsym; +} + +void UpdatePPMdContext1(PPMdContext *self, PPMdCoreModel *model, PPMdState *state) +{ + state->Freq += 4; + self->SummFreq += 4; + + if(state[0].Freq > state[-1].Freq) + { + SWAP(state[0], state[-1]); + model->FoundState = &state[-1]; + if(state[-1].Freq > MAX_FREQ) model->RescalePPMdContext(self, model); + } + else + { + model->FoundState = state; + } +} + +void PPMdDecodeSymbol2(PPMdContext *self, PPMdCoreModel *model, SEE2Context *see) +{ + int n = self->LastStateIndex - model->LastMaskIndex; + PPMdState *ps[256]; + + int total = 0; + PPMdState *state = PPMdContextStates(self, model); + for(int i = 0; i < n; i++) + { + while(model->CharMask[state->Symbol] == model->EscCount) state++; + + total += state->Freq; + ps[i] = state++; + } + + model->scale += total; + int count = PPMdRangeCoderCurrentCount(&model->coder, model->scale); + + if(count < total) + { + int i = 0, highcount = ps[0]->Freq; + while(highcount <= count && i + 1 < n) highcount += ps[++i]->Freq; + + RemovePPMdRangeCoderSubRange(&model->coder, highcount - ps[i]->Freq, highcount); + UpdateSEE2(see); + UpdatePPMdContext2(self, model, ps[i]); + } + else + { + RemovePPMdRangeCoderSubRange(&model->coder, total, model->scale); + model->LastMaskIndex = self->LastStateIndex; + see->Summ += model->scale; + + for(int i = 0; i < n; i++) model->CharMask[ps[i]->Symbol] = model->EscCount; + } +} + +void UpdatePPMdContext2(PPMdContext *self, PPMdCoreModel *model, PPMdState *state) +{ + model->FoundState = state; + state->Freq += 4; + self->SummFreq += 4; + if(state->Freq > MAX_FREQ) model->RescalePPMdContext(self, model); + model->EscCount++; + model->RunLength = model->InitRL; +} + +void RescalePPMdContext(PPMdContext *self, PPMdCoreModel *model) +{ + PPMdState *states = PPMdContextStates(self, model); + int n = self->LastStateIndex + 1; + + // Bump frequency of found state + model->FoundState->Freq += 4; + + // Divide all frequencies and sort list + int escfreq = self->SummFreq + 4; + int adder = (model->OrderFall == 0 ? 0 : 1); + self->SummFreq = 0; + + for(int i = 0; i < n; i++) + { + escfreq -= states[i].Freq; + states[i].Freq = (states[i].Freq + adder) >> 1; + self->SummFreq += states[i].Freq; + + // Keep states sorted by decreasing frequency + if(i > 0 && states[i].Freq > states[i - 1].Freq) + { + // If not sorted, move current state upwards until list is sorted + PPMdState tmp = states[i]; + + int j = i - 1; + while(j > 0 && tmp.Freq > states[j - 1].Freq) j--; + + memmove(&states[j + 1], &states[j], sizeof(PPMdState) * (i - j)); + states[j] = tmp; + } + } + + // TODO: add better sorting stage here. + + // Drop states whose frequency has fallen to 0 + if(states[n - 1].Freq == 0) + { + int numzeros = 1; + while(numzeros < n && states[n - 1 - numzeros].Freq == 0) numzeros++; + + escfreq += numzeros; + + self->LastStateIndex -= numzeros; + if(self->LastStateIndex == 0) + { + PPMdState tmp = states[0]; + do + { + tmp.Freq = (tmp.Freq + 1) >> 1; + escfreq >>= 1; + } while(escfreq > 1); + + FreeUnits(model->alloc, self->States, (n + 1) >> 1); + model->FoundState = PPMdContextOneState(self); + *model->FoundState = tmp; + + return; + } + + int n0 = (n + 1) >> 1, n1 = (self->LastStateIndex + 2) >> 1; + if(n0 != n1) self->States = ShrinkUnits(model->alloc, self->States, n0, n1); + } + + self->SummFreq += (escfreq + 1) >> 1; + + // The found state is the first one to breach the limit, thus it is the largest and also first + model->FoundState = PPMdContextStates(self, model); +} + +void ClearPPMdModelMask(PPMdCoreModel *self) +{ + self->EscCount = 1; + memset(self->CharMask, 0, sizeof(self->CharMask)); +} diff --git a/ppmd/Context.h b/ppmd/Context.h new file mode 100644 index 0000000..54f6468 --- /dev/null +++ b/ppmd/Context.h @@ -0,0 +1,110 @@ +/* + * Context.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __PPMD_CONTEXT_H__ +#define __PPMD_CONTEXT_H__ + +#include "RangeCoder.h" +#include "SubAllocator.h" + +#include +#include + +#define MAX_O 255 +#define INT_BITS 7 +#define PERIOD_BITS 7 +#define TOT_BITS (INT_BITS + PERIOD_BITS) +#define MAX_FREQ 124 +#define INTERVAL (1 << INT_BITS) +#define BIN_SCALE (1 << TOT_BITS) + +#define SWAP(t1, t2) \ + { \ + PPMdState tmp = (t1); \ + (t1) = (t2); \ + (t2) = tmp; \ + } + +typedef struct SEE2Context +{ // SEE-contexts for PPM-contexts with masked symbols + uint16_t Summ; + uint8_t Shift, Count; +} __attribute__((__packed__)) SEE2Context; + +typedef struct PPMdContext PPMdContext; + +typedef struct PPMdState +{ + uint8_t Symbol, Freq; + uint32_t Successor; +} __attribute__((__packed__)) PPMdState; + +struct PPMdContext +{ + uint8_t LastStateIndex, Flags; + uint16_t SummFreq; + uint32_t States; + uint32_t Suffix; +} __attribute__((__packed__)); + +typedef struct PPMdCoreModel PPMdCoreModel; + +struct PPMdCoreModel +{ + PPMdSubAllocator *alloc; + + PPMdRangeCoder coder; + uint32_t scale; + + PPMdState *FoundState; // found next state transition + int OrderFall, InitEsc, RunLength, InitRL; + uint8_t CharMask[256]; + uint8_t LastMaskIndex, EscCount, PrevSuccess; + + void (*RescalePPMdContext)(PPMdContext *self, PPMdCoreModel *model); +}; + +SEE2Context MakeSEE2(int initval, int count); +unsigned int GetSEE2MeanMasked(SEE2Context *self); +unsigned int GetSEE2Mean(SEE2Context *self); +void UpdateSEE2(SEE2Context *self); + +PPMdContext *PPMdStateSuccessor(PPMdState *self, PPMdCoreModel *model); +void SetPPMdStateSuccessorPointer(PPMdState *self, PPMdContext *newsuccessor, PPMdCoreModel *model); +PPMdState *PPMdContextStates(PPMdContext *self, PPMdCoreModel *model); +void SetPPMdContextStatesPointer(PPMdContext *self, PPMdState *newstates, PPMdCoreModel *model); +PPMdContext *PPMdContextSuffix(PPMdContext *self, PPMdCoreModel *model); +void SetPPMdContextSuffixPointer(PPMdContext *self, PPMdContext *newsuffix, PPMdCoreModel *model); +PPMdState *PPMdContextOneState(PPMdContext *self); + +PPMdContext *NewPPMdContext(PPMdCoreModel *model); +PPMdContext *NewPPMdContextAsChildOf(PPMdCoreModel *model, PPMdContext *suffixcontext, PPMdState *suffixstate, + PPMdState *firststate); + +void PPMdDecodeBinSymbol(PPMdContext *self, PPMdCoreModel *model, uint16_t *bs, int freqlimit, bool altnextbit); +int PPMdDecodeSymbol1(PPMdContext *self, PPMdCoreModel *model, bool greaterorequal); +void UpdatePPMdContext1(PPMdContext *self, PPMdCoreModel *model, PPMdState *state); +void PPMdDecodeSymbol2(PPMdContext *self, PPMdCoreModel *model, SEE2Context *see); +void UpdatePPMdContext2(PPMdContext *self, PPMdCoreModel *model, PPMdState *state); +void RescalePPMdContext(PPMdContext *self, PPMdCoreModel *model); + +void ClearPPMdModelMask(PPMdCoreModel *self); + +#endif diff --git a/ppmd/RangeCoder.c b/ppmd/RangeCoder.c new file mode 100644 index 0000000..5e62cdd --- /dev/null +++ b/ppmd/RangeCoder.c @@ -0,0 +1,109 @@ +/* + * RangeCoder.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "RangeCoder.h" + +void InitializePPMdRangeCoder(PPMdRangeCoder *self, PPMdReadFunction *readfunc, void *inputcontext, bool uselow, + int bottom) +{ + self->readfunc = readfunc; + self->inputcontext = inputcontext; + self->low = 0; + self->code = 0; + self->range = 0xffffffff; + self->uselow = uselow; + self->bottom = bottom; + for(int i = 0; i < 4; i++) self->code = (self->code << 8) | readfunc(inputcontext); +} + +uint32_t PPMdRangeCoderCurrentCount(PPMdRangeCoder *self, uint32_t scale) +{ + self->range /= scale; + return (self->code - self->low) / self->range; +} + +void RemovePPMdRangeCoderSubRange(PPMdRangeCoder *self, uint32_t lowcount, uint32_t highcount) +{ + if(self->uselow) + self->low += self->range * lowcount; + else + self->code -= self->range * lowcount; + + self->range *= highcount - lowcount; + + NormalizePPMdRangeCoder(self); +} + +int NextWeightedBitFromPPMdRangeCoder(PPMdRangeCoder *self, int weight, int size) +{ + uint32_t val = PPMdRangeCoderCurrentCount(self, size); + + if(val < weight) + { + RemovePPMdRangeCoderSubRange(self, 0, weight); + return 0; + } + else + { + RemovePPMdRangeCoderSubRange(self, weight, size); + return 1; + } +} + +int NextWeightedBitFromPPMdRangeCoder2(PPMdRangeCoder *self, int weight, int shift) +{ + uint32_t threshold = (self->range >> shift) * weight; + + int bit; + if(self->code < threshold) // <= ? + { + bit = 0; + self->range = threshold; + } + else + { + bit = 1; + self->range -= threshold; + self->code -= threshold; + } + + NormalizePPMdRangeCoder(self); + + return bit; +} + +void NormalizePPMdRangeCoder(PPMdRangeCoder *self) +{ + for(;;) + { + if((self->low ^ (self->low + self->range)) >= 0x1000000) + { + if(self->range >= self->bottom) + break; + else + self->range = -self->low & (self->bottom - 1); + } + + int byte = self->readfunc(self->inputcontext); + self->code = (self->code << 8) | byte; + self->range <<= 8; + self->low <<= 8; + } +} diff --git a/ppmd/RangeCoder.h b/ppmd/RangeCoder.h new file mode 100644 index 0000000..2652c99 --- /dev/null +++ b/ppmd/RangeCoder.h @@ -0,0 +1,50 @@ +/* + * RangeCoder.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __PPMD_RANGE_CODER_H__ +#define __PPMD_RANGE_CODER_H__ + +#include +#include + +typedef int PPMdReadFunction(void *context); + +typedef struct PPMdRangeCoder +{ + PPMdReadFunction *readfunc; + void *inputcontext; + + uint32_t low, code, range, bottom; + bool uselow; +} PPMdRangeCoder; + +void InitializePPMdRangeCoder(PPMdRangeCoder *self, PPMdReadFunction *readfunc, void *inputcontext, bool uselow, + int bottom); + +uint32_t PPMdRangeCoderCurrentCount(PPMdRangeCoder *self, uint32_t scale); +void RemovePPMdRangeCoderSubRange(PPMdRangeCoder *self, uint32_t lowcount, uint32_t highcount); + +int NextWeightedBitFromPPMdRangeCoder(PPMdRangeCoder *self, int weight, int size); + +int NextWeightedBitFromPPMdRangeCoder2(PPMdRangeCoder *self, int weight, int shift); + +void NormalizePPMdRangeCoder(PPMdRangeCoder *self); + +#endif diff --git a/ppmd/SubAllocator.h b/ppmd/SubAllocator.h new file mode 100644 index 0000000..0000f5d --- /dev/null +++ b/ppmd/SubAllocator.h @@ -0,0 +1,69 @@ +/* + * SubAllocator.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __PPMD_SUB_ALLOCATOR_H__ +#define __PPMD_SUB_ALLOCATOR_H__ + +#include +#include +#include + +typedef struct PPMdSubAllocator PPMdSubAllocator; + +struct PPMdSubAllocator +{ + void (*Init)(PPMdSubAllocator *self); + uint32_t (*AllocContext)(PPMdSubAllocator *self); + uint32_t (*AllocUnits)(PPMdSubAllocator *self, int num); // 1 unit == 12 bytes, NU <= 128 + uint32_t (*ExpandUnits)(PPMdSubAllocator *self, uint32_t oldoffs, int oldnum); + uint32_t (*ShrinkUnits)(PPMdSubAllocator *self, uint32_t oldoffs, int oldnum, int newnum); + void (*FreeUnits)(PPMdSubAllocator *self, uint32_t offs, int num); +}; + +static inline void InitSubAllocator(PPMdSubAllocator *self) { self->Init(self); }; + +static inline uint32_t AllocContext(PPMdSubAllocator *self) { return self->AllocContext(self); } + +static inline uint32_t AllocUnits(PPMdSubAllocator *self, int num) { return self->AllocUnits(self, num); } + +static inline uint32_t ExpandUnits(PPMdSubAllocator *self, uint32_t oldoffs, int oldnum) +{ return self->ExpandUnits(self, oldoffs, oldnum); } + +static inline uint32_t ShrinkUnits(PPMdSubAllocator *self, uint32_t oldoffs, int oldnum, int newnum) +{ return self->ShrinkUnits(self, oldoffs, oldnum, newnum); } + +static inline void FreeUnits(PPMdSubAllocator *self, uint32_t offs, int num) +{ return self->FreeUnits(self, offs, num); } + +// TODO: Keep pointers as pointers on 32 bit, and offsets on 64 bit. + +static inline void *OffsetToPointer(void *base, uint32_t offset) +{ + if(!offset) return NULL; + return ((uint8_t *)base) + offset; +} + +static inline uint32_t PointerToOffset(void *base, void *pointer) +{ + if(!pointer) return 0; + return (uint32_t)(((uintptr_t)pointer) - (uintptr_t)base); +} + +#endif diff --git a/ppmd/SubAllocatorVariantI.c b/ppmd/SubAllocatorVariantI.c new file mode 100644 index 0000000..90b7e1b --- /dev/null +++ b/ppmd/SubAllocatorVariantI.c @@ -0,0 +1,365 @@ +/* + * SubAllocatorVariantI.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "SubAllocatorVariantI.h" + +#include +#include + +#define N1 4 +#define N2 4 +#define N3 4 +#define N4 ((128 + 3 - 1 * N1 - 2 * N2 - 3 * N3) / 4) +#define UNIT_SIZE 12 +#define N_INDEXES (N1 + N2 + N3 + N4) + +static PPMdMemoryBlockVariantI *NextBlock(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc); +static void SetNextBlock(PPMdMemoryBlockVariantI *self, PPMdMemoryBlockVariantI *newnext, + PPMdSubAllocatorVariantI *alloc); +static bool AreBlocksAvailable(PPMdMemoryBlockVariantI *self); +static void LinkBlockAfter(PPMdMemoryBlockVariantI *self, PPMdMemoryBlockVariantI *p, PPMdSubAllocatorVariantI *alloc); +static void UnlinkBlockAfter(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc); +static void *RemoveBlockAfter(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc); +static void InsertBlockAfter(PPMdMemoryBlockVariantI *self, void *pv, int NU, PPMdSubAllocatorVariantI *alloc); + +static unsigned int I2B(PPMdSubAllocatorVariantI *self, int index); +static void SplitBlock(PPMdSubAllocatorVariantI *self, void *pv, int oldindex, int newindex); +// static uint32_t GetUsedMemory(PPMdSubAllocatorVariantI *self); + +static void InitVariantI(PPMdSubAllocatorVariantI *self); +static uint32_t AllocContextVariantI(PPMdSubAllocatorVariantI *self); +static uint32_t AllocUnitsVariantI(PPMdSubAllocatorVariantI *self, int num); +static uint32_t _AllocUnits(PPMdSubAllocatorVariantI *self, int index); +static uint32_t ExpandUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int oldnum); +static uint32_t ShrinkUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int oldnum, int newnum); +static void FreeUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t offs, int num); + +static inline void GlueFreeBlocks(PPMdSubAllocatorVariantI *self); + +PPMdSubAllocatorVariantI *CreateSubAllocatorVariantI(int size) +{ + PPMdSubAllocatorVariantI *self = malloc(sizeof(PPMdSubAllocatorVariantI) + size); + if(!self) return NULL; + + self->core.Init = (void *)InitVariantI; + self->core.AllocContext = (void *)AllocContextVariantI; + self->core.AllocUnits = (void *)AllocUnitsVariantI; + self->core.ExpandUnits = (void *)ExpandUnitsVariantI; + self->core.ShrinkUnits = (void *)ShrinkUnitsVariantI; + self->core.FreeUnits = (void *)FreeUnitsVariantI; + + self->SubAllocatorSize = size; + + return self; +} + +void FreeSubAllocatorVariantI(PPMdSubAllocatorVariantI *self) { free(self); } + +static void InitVariantI(PPMdSubAllocatorVariantI *self) +{ + memset(self->BList, 0, sizeof(self->BList)); + + self->pText = self->HeapStart; + self->HighUnit = self->HeapStart + self->SubAllocatorSize; + unsigned int diff = UNIT_SIZE * (self->SubAllocatorSize / 8 / UNIT_SIZE * 7); + self->LowUnit = self->UnitsStart = self->HighUnit - diff; + self->GlueCount = 0; + + for(int i = 0; i < N1; i++) self->Index2Units[i] = 1 + i; + for(int i = 0; i < N2; i++) self->Index2Units[N1 + i] = 2 + N1 + i * 2; + for(int i = 0; i < N3; i++) self->Index2Units[N1 + N2 + i] = 3 + N1 + 2 * N2 + i * 3; + for(int i = 0; i < N4; i++) self->Index2Units[N1 + N2 + N3 + i] = 4 + N1 + 2 * N2 + 3 * N3 + i * 4; + + int i = 0; + for(int k = 0; k < 128; k++) + { + if(self->Index2Units[i] < k + 1) i++; + self->Units2Index[k] = i; + } +} + +static uint32_t AllocContextVariantI(PPMdSubAllocatorVariantI *self) +{ + if(self->HighUnit != self->LowUnit) + { + self->HighUnit -= UNIT_SIZE; + return PointerToOffset(self, self->HighUnit); + } + else if(AreBlocksAvailable(&self->BList[0])) + return PointerToOffset(self, RemoveBlockAfter(&self->BList[0], self)); + else + return _AllocUnits(self, 0); +} + +static uint32_t AllocUnitsVariantI(PPMdSubAllocatorVariantI *self, int num) +{ + int index = self->Units2Index[num - 1]; + + if(AreBlocksAvailable(&self->BList[index])) + return PointerToOffset(self, RemoveBlockAfter(&self->BList[index], self)); + + void *units = self->LowUnit; + self->LowUnit += I2B(self, index); + if(self->LowUnit <= self->HighUnit) return PointerToOffset(self, units); + + self->LowUnit -= I2B(self, index); + + return _AllocUnits(self, index); +} + +static uint32_t _AllocUnits(PPMdSubAllocatorVariantI *self, int index) +{ + if(self->GlueCount == 0) + { + GlueFreeBlocks(self); + if(AreBlocksAvailable(&self->BList[index])) + return PointerToOffset(self, RemoveBlockAfter(&self->BList[index], self)); + } + + for(int i = index + 1; i < N_INDEXES; i++) + { + if(AreBlocksAvailable(&self->BList[i])) + { + void *units = RemoveBlockAfter(&self->BList[i], self); + SplitBlock(self, units, i, index); + return PointerToOffset(self, units); + } + } + + self->GlueCount--; + + int i = I2B(self, index); + if(self->UnitsStart - self->pText > i) + { + self->UnitsStart -= i; + return PointerToOffset(self, self->UnitsStart); + } + + return 0; +} + +static uint32_t ExpandUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int oldnum) +{ + void *oldptr = OffsetToPointer(self, oldoffs); + int oldindex = self->Units2Index[oldnum - 1]; + int newindex = self->Units2Index[oldnum]; + if(oldindex == newindex) return oldoffs; + + uint32_t offs = AllocUnitsVariantI(self, oldnum + 1); + if(offs) + { + memcpy(OffsetToPointer(self, offs), oldptr, oldnum * UNIT_SIZE); + InsertBlockAfter(&self->BList[oldindex], oldptr, oldnum, self); + } + return offs; +} + +static uint32_t ShrinkUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int oldnum, int newnum) +{ + void *oldptr = OffsetToPointer(self, oldoffs); + int oldindex = self->Units2Index[oldnum - 1]; + int newindex = self->Units2Index[newnum - 1]; + if(oldindex == newindex) return oldoffs; + + if(AreBlocksAvailable(&self->BList[newindex])) + { + void *ptr = RemoveBlockAfter(&self->BList[newindex], self); + memcpy(ptr, oldptr, newnum * UNIT_SIZE); + InsertBlockAfter(&self->BList[oldindex], oldptr, self->Index2Units[oldindex], self); + return PointerToOffset(self, ptr); + } + else + { + SplitBlock(self, oldptr, oldindex, newindex); + return oldoffs; + } +} + +static void FreeUnitsVariantI(PPMdSubAllocatorVariantI *self, uint32_t offs, int num) +{ + int index = self->Units2Index[num - 1]; + InsertBlockAfter(&self->BList[index], OffsetToPointer(self, offs), self->Index2Units[index], self); +} + +uint32_t GetUsedMemoryVariantI(PPMdSubAllocatorVariantI *self) +{ + size_t size = self->SubAllocatorSize - (self->HighUnit - self->LowUnit) - (self->UnitsStart - self->pText); + + for(int i = 0; i < N_INDEXES; i++) size -= UNIT_SIZE * self->Index2Units[i] * self->BList[i].Stamp; + + return (uint32_t)size; +} + +void SpecialFreeUnitVariantI(PPMdSubAllocatorVariantI *self, uint32_t offs) +{ + void *ptr = OffsetToPointer(self, offs); + if((uint8_t *)ptr == self->UnitsStart) + { + *(uint32_t *)ptr = 0xffffffff; + self->UnitsStart += UNIT_SIZE; + } + else + InsertBlockAfter(&self->BList[0], ptr, 1, self); +} + +uint32_t MoveUnitsUpVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int num) +{ + void *oldptr = OffsetToPointer(self, oldoffs); + int index = self->Units2Index[num - 1]; + + if((uint8_t *)oldptr > self->UnitsStart + 16 * 1024 || oldoffs > self->BList[index].next) return oldoffs; + + void *ptr = RemoveBlockAfter(&self->BList[index], self); + memcpy(ptr, oldptr, num * UNIT_SIZE); + + int newnum = self->Index2Units[index]; + if((uint8_t *)oldptr != self->UnitsStart) + InsertBlockAfter(&self->BList[index], oldptr, newnum, self); + else + self->UnitsStart += newnum * UNIT_SIZE; + + return PointerToOffset(self, ptr); +} + +void ExpandTextAreaVariantI(PPMdSubAllocatorVariantI *self) +{ + PPMdMemoryBlockVariantI *p; + unsigned int Count[N_INDEXES]; + memset(Count, 0, sizeof(Count)); + + while((p = (PPMdMemoryBlockVariantI *)self->UnitsStart)->Stamp == 0xffffffff) + { + PPMdMemoryBlockVariantI *pm = p; + self->UnitsStart = (uint8_t *)(pm + pm->NU); + Count[self->Units2Index[pm->NU - 1]]++; + pm->Stamp = 0; + } + + for(int i = 0; i < N_INDEXES; i++) + for(p = &self->BList[i]; Count[i] != 0; p = NextBlock(p, self)) + while(!NextBlock(p, self)->Stamp) + { + UnlinkBlockAfter(p, self); + self->BList[i].Stamp--; + if(!--Count[i]) break; + } +} + +static inline void GlueFreeBlocks(PPMdSubAllocatorVariantI *self) +{ + if(self->LowUnit != self->HighUnit) *self->LowUnit = 0; + + PPMdMemoryBlockVariantI s0, *p0 = &s0; + s0.next = 0; + for(int i = 0; i < N_INDEXES; i++) + { + while(AreBlocksAvailable(&self->BList[i])) + { + PPMdMemoryBlockVariantI *p = (PPMdMemoryBlockVariantI *)RemoveBlockAfter(&self->BList[i], self); + if(!p->NU) continue; + PPMdMemoryBlockVariantI *p1; + while((p1 = p + p->NU)->Stamp == 0xffffffff) + { + p->NU += p1->NU; + p1->NU = 0; + } + LinkBlockAfter(p0, p, self); + p0 = p; + } + } + + while(AreBlocksAvailable(&s0)) + { + PPMdMemoryBlockVariantI *p = RemoveBlockAfter(&s0, self); + int sz = p->NU; + if(!sz) continue; + + while(sz > 128) + { + InsertBlockAfter(&self->BList[N_INDEXES - 1], p, 128, self); + sz -= 128; + p += 128; + } + + int i = self->Units2Index[sz - 1]; + if(self->Index2Units[i] != sz) + { + i--; + int k = sz - self->Index2Units[i]; + InsertBlockAfter(&self->BList[k - 1], p + (sz - k), k, self); + } + InsertBlockAfter(&self->BList[i], p, self->Index2Units[i], self); + } + self->GlueCount = 1 << 13; +} + +static PPMdMemoryBlockVariantI *NextBlock(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc) +{ return OffsetToPointer(&alloc->core, self->next); } + +static void SetNextBlock(PPMdMemoryBlockVariantI *self, PPMdMemoryBlockVariantI *newnext, + PPMdSubAllocatorVariantI *alloc) +{ self->next = PointerToOffset(&alloc->core, newnext); } + +static bool AreBlocksAvailable(PPMdMemoryBlockVariantI *self) { return self->next != 0; } + +static void LinkBlockAfter(PPMdMemoryBlockVariantI *self, PPMdMemoryBlockVariantI *p, PPMdSubAllocatorVariantI *alloc) +{ + SetNextBlock(p, NextBlock(self, alloc), alloc); + SetNextBlock(self, p, alloc); +} + +static void UnlinkBlockAfter(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc) +{ SetNextBlock(self, NextBlock(NextBlock(self, alloc), alloc), alloc); } + +static void *RemoveBlockAfter(PPMdMemoryBlockVariantI *self, PPMdSubAllocatorVariantI *alloc) +{ + PPMdMemoryBlockVariantI *p = NextBlock(self, alloc); + UnlinkBlockAfter(self, alloc); + self->Stamp--; + return p; +} + +static void InsertBlockAfter(PPMdMemoryBlockVariantI *self, void *pv, int NU, PPMdSubAllocatorVariantI *alloc) +{ + PPMdMemoryBlockVariantI *p = (PPMdMemoryBlockVariantI *)pv; + LinkBlockAfter(self, p, alloc); + p->Stamp = 0xffffffff; + p->NU = NU; + self->Stamp++; +} + +static inline unsigned int I2B(PPMdSubAllocatorVariantI *self, int index) +{ return UNIT_SIZE * self->Index2Units[index]; } + +static void SplitBlock(PPMdSubAllocatorVariantI *self, void *pv, int oldindex, int newindex) +{ + uint8_t *p = ((uint8_t *)pv) + I2B(self, newindex); + + int diff = self->Index2Units[oldindex] - self->Index2Units[newindex]; + int i = self->Units2Index[diff - 1]; + if(self->Index2Units[i] != diff) + { + int k = self->Index2Units[--i]; + InsertBlockAfter(&self->BList[i], p, k, self); + p += k * UNIT_SIZE; + diff -= k; + } + InsertBlockAfter(&self->BList[self->Units2Index[diff - 1]], p, diff, self); +} diff --git a/ppmd/SubAllocatorVariantI.h b/ppmd/SubAllocatorVariantI.h new file mode 100644 index 0000000..c27589b --- /dev/null +++ b/ppmd/SubAllocatorVariantI.h @@ -0,0 +1,52 @@ +/* + * SubAllocatorVariantI.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __PPMD_SUB_ALLOCATOR_VARIANT_I_H__ +#define __PPMD_SUB_ALLOCATOR_VARIANT_I_H__ + +#include "SubAllocator.h" + +typedef struct PPMdMemoryBlockVariantI +{ + uint32_t Stamp; + uint32_t next; + uint32_t NU; +} __attribute__((packed)) PPMdMemoryBlockVariantI; + +typedef struct PPMdSubAllocatorVariantI +{ + PPMdSubAllocator core; + + uint32_t GlueCount, SubAllocatorSize; + uint8_t Index2Units[38], Units2Index[128]; // constants + uint8_t *pText, *UnitsStart, *LowUnit, *HighUnit; + PPMdMemoryBlockVariantI BList[38]; + uint8_t HeapStart[0]; +} PPMdSubAllocatorVariantI; + +PPMdSubAllocatorVariantI *CreateSubAllocatorVariantI(int size); +void FreeSubAllocatorVariantI(PPMdSubAllocatorVariantI *self); + +uint32_t GetUsedMemoryVariantI(PPMdSubAllocatorVariantI *self); +void SpecialFreeUnitVariantI(PPMdSubAllocatorVariantI *self, uint32_t offs); +uint32_t MoveUnitsUpVariantI(PPMdSubAllocatorVariantI *self, uint32_t oldoffs, int num); +void ExpandTextAreaVariantI(PPMdSubAllocatorVariantI *self); + +#endif diff --git a/ppmd/VariantI.c b/ppmd/VariantI.c new file mode 100644 index 0000000..db429c4 --- /dev/null +++ b/ppmd/VariantI.c @@ -0,0 +1,886 @@ +/* + * VariantI.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "VariantI.h" + +#include + +#define UP_FREQ 5 +#define O_BOUND 9 + +static void RestartModel(PPMdModelVariantI *self); + +static void UpdateModel(PPMdModelVariantI *self, PPMdContext *mincontext); +static PPMdContext *CreateSuccessors(PPMdModelVariantI *self, bool skip, PPMdState *p1, PPMdContext *mincontext); +static PPMdContext *ReduceOrder(PPMdModelVariantI *self, PPMdState *state, PPMdContext *startcontext); +static void RestoreModel(PPMdModelVariantI *self, PPMdContext *currcontext, PPMdContext *mincontext, + PPMdContext *FSuccessor); + +static void ShrinkContext(PPMdContext *self, int newlastindex, bool scale, PPMdModelVariantI *model); +static PPMdContext *CutOffContext(PPMdContext *self, int order, PPMdModelVariantI *model); +static PPMdContext *RemoveBinConts(PPMdContext *self, int order, PPMdModelVariantI *model); + +static void DecodeBinSymbolVariantI(PPMdContext *self, PPMdModelVariantI *model); +static void DecodeSymbol1VariantI(PPMdContext *self, PPMdModelVariantI *model); +static void DecodeSymbol2VariantI(PPMdContext *self, PPMdModelVariantI *model); + +static void RescalePPMdContextVariantI(PPMdContext *self, PPMdModelVariantI *model); + +void StartPPMdModelVariantI(PPMdModelVariantI *self, PPMdReadFunction *readfunc, void *inputcontext, + PPMdSubAllocatorVariantI *alloc, int maxorder, int restoration) +{ + InitializePPMdRangeCoder(&self->core.coder, readfunc, inputcontext, true, 0x8000); + + if(maxorder < 2) // TODO: solid mode + { + memset(self->core.CharMask, 0, sizeof(self->core.CharMask)); + self->core.OrderFall = self->MaxOrder; + for(PPMdContext *pc = self->MaxContext; pc->Suffix; pc = PPMdContextSuffix(pc, &self->core)) + self->core.OrderFall--; + return; + } + + self->alloc = alloc; + self->core.alloc = &alloc->core; + + self->core.RescalePPMdContext = (void *)RescalePPMdContextVariantI; + + self->MaxOrder = maxorder; + self->MRMethod = restoration; + self->core.EscCount = 1; + + self->NS2BSIndx[0] = 2 * 0; + self->NS2BSIndx[1] = 2 * 1; + for(int i = 2; i < 11; i++) self->NS2BSIndx[i] = 2 * 2; + for(int i = 11; i < 256; i++) self->NS2BSIndx[i] = 2 * 3; + + for(int i = 0; i < UP_FREQ; i++) self->QTable[i] = i; + int m = UP_FREQ, k = 1, step = 1; + for(int i = UP_FREQ; i < 260; i++) + { + self->QTable[i] = m; + if(!--k) + { + m++; + step++; + k = step; + } + } + + self->DummySEE2Cont.Summ = 0xaf8f; + // self->DummySEE2Cont.Shift=0xac; + self->DummySEE2Cont.Count = 0x84; + self->DummySEE2Cont.Shift = PERIOD_BITS; + + self->endofstream = false; + + RestartModel(self); +} + +static void RestartModel(PPMdModelVariantI *self) +{ + InitSubAllocator(self->core.alloc); + + memset(self->core.CharMask, 0, sizeof(self->core.CharMask)); + + self->core.PrevSuccess = 0; + self->core.OrderFall = self->MaxOrder; + self->core.RunLength = self->core.InitRL = -((self->MaxOrder < 12) ? self->MaxOrder : 12) - 1; + + self->MaxContext = NewPPMdContext(&self->core); + self->MaxContext->LastStateIndex = 255; + self->MaxContext->SummFreq = 257; + self->MaxContext->States = AllocUnits(self->core.alloc, 256 / 2); + + PPMdState *maxstates = PPMdContextStates(self->MaxContext, &self->core); + for(int i = 0; i < 256; i++) + { + maxstates[i].Symbol = i; + maxstates[i].Freq = 1; + maxstates[i].Successor = 0; + } + + static const uint16_t InitBinEsc[8] = {0x3cdd, 0x1f3f, 0x59bf, 0x48f3, 0x64a1, 0x5abc, 0x6632, 0x6051}; + + int i = 0; + for(int m = 0; m < 25; m++) + { + while(self->QTable[i] == m) i++; + for(int k = 0; k < 8; k++) self->BinSumm[m][k] = BIN_SCALE - InitBinEsc[k] / (i + 1); + for(int k = 8; k < 64; k += 8) memcpy(&self->BinSumm[m][k], &self->BinSumm[m][0], 8 * sizeof(uint16_t)); + } + + i = 0; + for(int m = 0; m < 24; m++) + { + while(self->QTable[i + 3] == m + 3) i++; + for(int k = 0; k < 32; k++) self->SEE2Cont[m][k] = MakeSEE2(2 * i + 5, 7); + } +} + +int NextPPMdVariantIByte(PPMdModelVariantI *self) +{ + if(self->endofstream) return -1; + + PPMdContext *mincontext = self->MaxContext; + + if(mincontext->LastStateIndex != 0) + DecodeSymbol1VariantI(mincontext, self); + else + DecodeBinSymbolVariantI(mincontext, self); + + while(!self->core.FoundState) + { + do + { + self->core.OrderFall++; + mincontext = PPMdContextSuffix(mincontext, &self->core); + if(!mincontext) + { + self->endofstream = true; + return -1; + } + } while(mincontext->LastStateIndex == self->core.LastMaskIndex); + + DecodeSymbol2VariantI(mincontext, self); + } + + uint8_t byte = self->core.FoundState->Symbol; + + if(self->core.OrderFall == 0 && + (uint8_t *)PPMdStateSuccessor(self->core.FoundState, &self->core) >= self->alloc->UnitsStart) + { + self->MaxContext = PPMdStateSuccessor(self->core.FoundState, &self->core); + // PrefetchData(MaxContext) + } + else + { + UpdateModel(self, mincontext); + // PrefetchData(MaxContext) + if(self->core.EscCount == 0) ClearPPMdModelMask(&self->core); + } + + return byte; +} + +static void UpdateModel(PPMdModelVariantI *self, PPMdContext *mincontext) +{ + PPMdState fs = *self->core.FoundState; + PPMdState *state = NULL; + PPMdContext *currcontext = self->MaxContext; + + if(fs.Freq < MAX_FREQ / 4 && mincontext->Suffix) + { + PPMdContext *context = PPMdContextSuffix(mincontext, &self->core); + if(context->LastStateIndex != 0) + { + state = PPMdContextStates(context, &self->core); + + if(state->Symbol != fs.Symbol) + { + do state++; + while(state->Symbol != fs.Symbol); + + if(state[0].Freq >= state[-1].Freq) + { + SWAP(state[0], state[-1]); + state--; + } + } + + if(state->Freq < MAX_FREQ - 9) + { + state->Freq += 2; + context->SummFreq += 2; + } + } + else + { + state = PPMdContextOneState(context); + if(state->Freq < 32) state->Freq++; + } + } + + if(self->core.OrderFall == 0 && fs.Successor) + { + PPMdContext *newsuccessor = CreateSuccessors(self, true, state, mincontext); + SetPPMdStateSuccessorPointer(self->core.FoundState, newsuccessor, &self->core); + if(!newsuccessor) goto RESTART_MODEL; + self->MaxContext = newsuccessor; + return; + } + + *self->alloc->pText++ = fs.Symbol; + PPMdContext *Successor = (PPMdContext *)self->alloc->pText; + + if(self->alloc->pText >= self->alloc->UnitsStart) goto RESTART_MODEL; + + if(fs.Successor) + { + if((uint8_t *)PPMdStateSuccessor(&fs, &self->core) < self->alloc->UnitsStart) + { + SetPPMdStateSuccessorPointer(&fs, CreateSuccessors(self, false, state, mincontext), &self->core); + } + } + else + { + SetPPMdStateSuccessorPointer(&fs, ReduceOrder(self, state, mincontext), &self->core); + } + + if(!fs.Successor) goto RESTART_MODEL; + + if(--self->core.OrderFall == 0) + { + Successor = PPMdStateSuccessor(&fs, &self->core); + if(self->MaxContext != mincontext) self->alloc->pText--; + } + else if(self->MRMethod > MRM_FREEZE) + { + Successor = PPMdStateSuccessor(&fs, &self->core); + self->alloc->pText = self->alloc->HeapStart; + self->core.OrderFall = 0; + } + + int minnum = mincontext->LastStateIndex + 1; + int s0 = mincontext->SummFreq - minnum - (fs.Freq - 1); + uint8_t flag = fs.Symbol >= 0x40 ? 8 : 0; + + for(; currcontext != mincontext; currcontext = PPMdContextSuffix(currcontext, &self->core)) + { + int currnum = currcontext->LastStateIndex + 1; + if(currnum != 1) + { + if((currnum & 1) == 0) + { + uint32_t states = ExpandUnits(self->core.alloc, currcontext->States, currnum >> 1); + if(!states) goto RESTART_MODEL; + currcontext->States = states; + } + if(3 * currnum - 1 < minnum) currcontext->SummFreq++; + } + else + { + PPMdState *states = OffsetToPointer(self->core.alloc, AllocUnits(self->core.alloc, 1)); + if(!states) goto RESTART_MODEL; + states[0] = *(PPMdContextOneState(currcontext)); + SetPPMdContextStatesPointer(currcontext, states, &self->core); + + if(states[0].Freq < MAX_FREQ / 4 - 1) + states[0].Freq *= 2; + else + states[0].Freq = MAX_FREQ - 4; + + currcontext->SummFreq = states[0].Freq + self->core.InitEsc + (minnum > 3 ? 1 : 0); + } + + unsigned int cf = 2 * fs.Freq * (currcontext->SummFreq + 6); + unsigned int sf = s0 + currcontext->SummFreq; + unsigned int freq; + + if(cf < 6 * sf) + { + if(cf >= 4 * sf) + freq = 3; + else if(cf > sf) + freq = 2; + else + freq = 1; + currcontext->SummFreq += 4; + } + else + { + if(cf > 15 * sf) + freq = 7; + else if(cf > 12 * sf) + freq = 6; + else if(cf > 9 * sf) + freq = 5; + else + freq = 4; + currcontext->SummFreq += freq; + } + + currcontext->LastStateIndex++; + PPMdState *currstates = PPMdContextStates(currcontext, &self->core); + PPMdState *new = &currstates[currcontext->LastStateIndex]; + SetPPMdStateSuccessorPointer(new, Successor, &self->core); + new->Symbol = fs.Symbol; + new->Freq = freq; + currcontext->Flags |= flag; + } + + self->MaxContext = PPMdStateSuccessor(&fs, &self->core); + + return; + +RESTART_MODEL: + RestoreModel(self, currcontext, mincontext, PPMdStateSuccessor(&fs, &self->core)); +} + +static PPMdContext *CreateSuccessors(PPMdModelVariantI *self, bool skip, PPMdState *state, PPMdContext *context) +{ + PPMdContext *upbranch = PPMdStateSuccessor(self->core.FoundState, &self->core); + PPMdState *statelist[MAX_O]; + uint8_t sym = self->core.FoundState->Symbol; + int n = 0; + + if(!skip) + { + statelist[n++] = self->core.FoundState; + if(!context->Suffix) goto skip; + } + + if(state) + { + context = PPMdContextSuffix(context, &self->core); + if(PPMdStateSuccessor(state, &self->core) != upbranch) + { + context = PPMdStateSuccessor(state, &self->core); + goto skip; + } + statelist[n++] = state; + if(!context->Suffix) goto skip; + } + + do + { + context = PPMdContextSuffix(context, &self->core); + if(context->LastStateIndex != 0) + { + state = PPMdContextStates(context, &self->core); + while(state->Symbol != sym) state++; + + if(state->Freq < MAX_FREQ - 9) + { + state->Freq++; + context->SummFreq++; + } + } + else + { + state = PPMdContextOneState(context); + state->Freq += (!PPMdContextSuffix(context, &self->core)->LastStateIndex & (state->Freq < 24)); + } + + if(PPMdStateSuccessor(state, &self->core) != upbranch) + { + context = PPMdStateSuccessor(state, &self->core); + break; + } + statelist[n++] = state; + } while(context->Suffix); + +skip: + + if(n == 0) return context; + + PPMdContext ct; + uint8_t newsym = *(uint8_t *)upbranch; + + ct.LastStateIndex = 0; + ct.Flags = 0; + if(sym >= 0x40) ct.Flags |= 0x10; + if(newsym >= 0x40) ct.Flags |= 0x08; + + PPMdState *onestate = PPMdContextOneState(&ct); + onestate->Symbol = newsym; + SetPPMdStateSuccessorPointer(onestate, (PPMdContext *)(((uint8_t *)upbranch) + 1), &self->core); + + if(context->LastStateIndex != 0) + { + state = PPMdContextStates(context, &self->core); + while(state->Symbol != newsym) state++; + + int cf = state->Freq - 1; + int s0 = context->SummFreq - context->LastStateIndex - cf; + + if(2 * cf <= s0) + { + if(5 * cf > s0) + onestate->Freq = 2; + else + onestate->Freq = 1; + } + else + onestate->Freq = 1 + ((cf + 2 * s0 - 3) / s0); + } + else + onestate->Freq = PPMdContextOneState(context)->Freq; + + for(int i = n - 1; i >= 0; i--) + { + PPMdContext *newcontext = (PPMdContext *)OffsetToPointer(self->core.alloc, AllocContext(self->core.alloc)); + if(!newcontext) return NULL; + + memcpy(newcontext, &ct, 8); + SetPPMdContextSuffixPointer(newcontext, context, &self->core); + SetPPMdStateSuccessorPointer(statelist[i], newcontext, &self->core); + + context = newcontext; + } + + return context; +} + +static PPMdContext *ReduceOrder(PPMdModelVariantI *self, PPMdState *state, PPMdContext *startcontext) +{ + PPMdState *statelist[MAX_O]; + PPMdContext *context = startcontext, *upbranch = (PPMdContext *)self->alloc->pText; + uint8_t sym = self->core.FoundState->Symbol; + + int n = 0; + + statelist[n++] = self->core.FoundState; + self->core.OrderFall++; + + if(state) + { + context = PPMdContextSuffix(context, &self->core); + if(state->Successor) goto skip; + statelist[n++] = state; + self->core.OrderFall++; + } + + for(;;) + { + if(!context->Suffix) + { + if(self->MRMethod > MRM_FREEZE) + { + for(int i = 0; i < n; i++) SetPPMdStateSuccessorPointer(statelist[i], context, &self->core); + self->alloc->pText = self->alloc->HeapStart + 1; + self->core.OrderFall = 1; + } + else + { + for(int i = 0; i < n; i++) SetPPMdStateSuccessorPointer(statelist[i], upbranch, &self->core); + } + return context; + } + + context = PPMdContextSuffix(context, &self->core); + + if(context->LastStateIndex) + { + state = PPMdContextStates(context, &self->core); + while(state->Symbol != sym) state++; + + if(state->Freq < MAX_FREQ - 9) + { + state->Freq += 2; + context->SummFreq += 2; + } + } + else + { + state = PPMdContextOneState(context); + if(state->Freq < 32) state->Freq++; + } + + if(state->Successor) break; + + statelist[n++] = state; + self->core.OrderFall++; + } +skip: + + if(self->MRMethod > MRM_FREEZE) + { + PPMdContext *successor = PPMdStateSuccessor(state, &self->core); + for(int i = 0; i < n; i++) SetPPMdStateSuccessorPointer(statelist[i], successor, &self->core); + + self->alloc->pText = self->alloc->HeapStart + 1; + self->core.OrderFall = 1; + + return successor; + } + else + { + for(int i = 0; i < n; i++) SetPPMdStateSuccessorPointer(statelist[i], upbranch, &self->core); + } + + if(PPMdStateSuccessor(state, &self->core) <= upbranch) + { + PPMdState *tmp = self->core.FoundState; + self->core.FoundState = state; + SetPPMdStateSuccessorPointer(state, CreateSuccessors(self, false, NULL, context), &self->core); + self->core.FoundState = tmp; + } + + if(self->core.OrderFall == 1 && startcontext == self->MaxContext) + { + self->core.FoundState->Successor = state->Successor; + self->alloc->pText--; + } + + return PPMdStateSuccessor(state, &self->core); +} + +static void RestoreModel(PPMdModelVariantI *self, PPMdContext *currcontext, PPMdContext *mincontext, + PPMdContext *FSuccessor) +{ + self->alloc->pText = self->alloc->HeapStart; + + PPMdContext *context = self->MaxContext; + while(context != currcontext) + { + if(context->LastStateIndex == 1) + { + PPMdState state = *(PPMdContextStates(context, &self->core)); + SpecialFreeUnitVariantI(self->alloc, context->States); + + state.Freq = (state.Freq + 11) >> 3; + *(PPMdContextOneState(context)) = state; + + context->LastStateIndex = 0; + context->Flags &= 0x10; + if(state.Symbol >= 0x40) context->Flags += 0x08; + } + else + { + ShrinkContext(context, context->LastStateIndex - 1, false, self); + } + + context = PPMdContextSuffix(context, &self->core); + } + + while(context != mincontext) + { + if(!context->LastStateIndex) + { + PPMdContextOneState(context)->Freq = (PPMdContextOneState(context)->Freq + 1) >> 1; + } + else + { + context->SummFreq += 4; + if(context->SummFreq > 128 + 4 * context->LastStateIndex) + ShrinkContext(context, context->LastStateIndex, true, self); + } + + context = PPMdContextSuffix(context, &self->core); + } + + if(self->MRMethod > MRM_FREEZE) + { + self->MaxContext = FSuccessor; + if(!(self->alloc->BList[1].Stamp & 1)) self->alloc->GlueCount++; + } + else if(self->MRMethod == MRM_FREEZE) + { + while(self->MaxContext->Suffix) self->MaxContext = PPMdContextSuffix(self->MaxContext, &self->core); + + RemoveBinConts(self->MaxContext, 0, self); + self->MRMethod = self->MRMethod + 1; + self->alloc->GlueCount = 0; + self->core.OrderFall = self->MaxOrder; + } + else if(self->MRMethod == MRM_RESTART || GetUsedMemoryVariantI(self->alloc) < (self->alloc->SubAllocatorSize >> 1)) + { + RestartModel(self); + self->core.EscCount = 0; + } + else + { + while(self->MaxContext->Suffix) self->MaxContext = PPMdContextSuffix(self->MaxContext, &self->core); + do + { + CutOffContext(self->MaxContext, 0, self); + ExpandTextAreaVariantI(self->alloc); + } while(GetUsedMemoryVariantI(self->alloc) > 3 * (self->alloc->SubAllocatorSize >> 2)); + + self->alloc->GlueCount = 0; + self->core.OrderFall = self->MaxOrder; + } +} + +static void ShrinkContext(PPMdContext *self, int newlastindex, bool scale, PPMdModelVariantI *model) +{ + self->States = + ShrinkUnits(model->core.alloc, self->States, (self->LastStateIndex + 2) >> 1, (newlastindex + 2) >> 1); + self->LastStateIndex = newlastindex; + + if(scale) + self->Flags &= 0x14; + else + self->Flags &= 0x10; + + PPMdState *states = PPMdContextStates(self, &model->core); + int escfreq = self->SummFreq; + self->SummFreq = 0; + + for(int i = 0; i <= self->LastStateIndex; i++) + { + escfreq -= states[i].Freq; + if(scale) states[i].Freq = (states[i].Freq + 1) >> 1; + self->SummFreq += states[i].Freq; + if(states[i].Symbol >= 0x40) self->Flags |= 0x08; + } + + if(scale) escfreq = (escfreq + 1) >> 1; + + self->SummFreq += escfreq; +} + +static PPMdContext *CutOffContext(PPMdContext *self, int order, PPMdModelVariantI *model) +{ + if(self->LastStateIndex == 0) + { + PPMdState *onestate = PPMdContextOneState(self); + if((uint8_t *)PPMdStateSuccessor(onestate, &model->core) >= model->alloc->UnitsStart) + { + if(order < model->MaxOrder) + { + // PrefetchData(p->Successor); + SetPPMdStateSuccessorPointer( + onestate, CutOffContext(PPMdStateSuccessor(onestate, &model->core), order + 1, model), + &model->core); + } + else + onestate->Successor = 0; + + if(!onestate->Successor && order > O_BOUND) + { + SpecialFreeUnitVariantI(model->alloc, PointerToOffset(model->core.alloc, self)); + return NULL; + } + + return self; + } + else + { + SpecialFreeUnitVariantI(model->alloc, PointerToOffset(model->core.alloc, self)); + return NULL; + } + } + // PrefetchData(self->States); + + int oldnum = (self->LastStateIndex + 2) >> 1; + self->States = MoveUnitsUpVariantI(model->alloc, self->States, oldnum); + + int n = self->LastStateIndex; + PPMdState *states = PPMdContextStates(self, &model->core); + for(int i = n; i >= 0; i--) + { + if((uint8_t *)PPMdStateSuccessor(&states[i], &model->core) < model->alloc->UnitsStart) + { + states[i].Successor = 0; + SWAP(states[i], states[n]); + n--; + } + else if(order < model->MaxOrder) + { + // PrefetchData(state->Successor); + SetPPMdStateSuccessorPointer(&states[i], + CutOffContext(PPMdStateSuccessor(&states[i], &model->core), order + 1, model), + &model->core); + } + else + states[i].Successor = 0; + } + + if(n != self->LastStateIndex && order) + { + if(n < 0) + { + FreeUnits(model->core.alloc, self->States, oldnum); + SpecialFreeUnitVariantI(model->alloc, PointerToOffset(model->core.alloc, self)); + return NULL; + } + else if(n == 0) + { + PPMdState state = *(PPMdContextStates(self, &model->core)); + FreeUnits(model->core.alloc, self->States, oldnum); + + state.Freq = (state.Freq + 11) >> 3; + *(PPMdContextOneState(self)) = state; + + self->LastStateIndex = 0; + self->Flags &= 0x10; + if(state.Symbol >= 0x40) self->Flags += 0x08; + } + else + ShrinkContext(self, n, self->SummFreq > 16 * n, model); + } + return self; +} + +static PPMdContext *RemoveBinConts(PPMdContext *self, int order, PPMdModelVariantI *model) +{ + if(self->LastStateIndex == 0) + { + PPMdState *state = PPMdContextOneState(self); + if((uint8_t *)PPMdStateSuccessor(state, &model->core) >= model->alloc->UnitsStart && order < model->MaxOrder) + { + // PrefetchData(onestate->Successor); + SetPPMdStateSuccessorPointer( + state, RemoveBinConts(PPMdStateSuccessor(state, &model->core), order + 1, model), &model->core); + } + else + state->Successor = 0; + + if(!state->Successor) + { + PPMdContext *suffix = PPMdContextSuffix(self, &model->core); + if(suffix->LastStateIndex == 0 || suffix->Flags == 0xff) + { + FreeUnits(model->core.alloc, PointerToOffset(model->core.alloc, self), 1); + return NULL; + } + } + + return self; + } + // PrefetchData(self->States); + + PPMdState *states = PPMdContextStates(self, &model->core); + for(int i = self->LastStateIndex; i >= 0; i--) + { + if((uint8_t *)PPMdStateSuccessor(&states[i], &model->core) >= model->alloc->UnitsStart && + order < model->MaxOrder) + { + // PrefetchData(states[i].Successor); + SetPPMdStateSuccessorPointer(&states[i], + RemoveBinConts(PPMdStateSuccessor(&states[i], &model->core), order + 1, model), + &model->core); + } + else + states[i].Successor = 0; + } + + return self; +} + +static void DecodeBinSymbolVariantI(PPMdContext *self, PPMdModelVariantI *model) +{ + PPMdState *rs = PPMdContextOneState(self); + + uint8_t index = + model->NS2BSIndx[PPMdContextSuffix(self, &model->core)->LastStateIndex] + model->core.PrevSuccess + self->Flags; + uint16_t *bs = &model->BinSumm[model->QTable[rs->Freq - 1]][index + ((model->core.RunLength >> 26) & 0x20)]; + + PPMdDecodeBinSymbol(self, &model->core, bs, 196, false); +} + +static void DecodeSymbol1VariantI(PPMdContext *self, PPMdModelVariantI *model) +{ PPMdDecodeSymbol1(self, &model->core, true); } + +static void DecodeSymbol2VariantI(PPMdContext *self, PPMdModelVariantI *model) +{ + SEE2Context *see; + + // uint8_t *pb=(uint8_t *)PPMdContextStates(self); + // unsigned int t=2*self->LastStateIndex; + // PrefetchData(pb); + // PrefetchData(pb+t); + // PrefetchData(pb+2*t); + // PrefetchData(pb+3*t); + + if(self->LastStateIndex != 255) + { + int n = PPMdContextSuffix(self, &model->core)->LastStateIndex; + see = &model->SEE2Cont[model->QTable[self->LastStateIndex + 2] - 3] + [(self->SummFreq > 11 * (self->LastStateIndex + 1) ? 1 : 0) + + (2 * self->LastStateIndex < n + model->core.LastMaskIndex ? 2 : 0) + self->Flags]; + model->core.scale = GetSEE2Mean(see); + } + else + { + model->core.scale = 1; + see = &model->DummySEE2Cont; + } + + PPMdDecodeSymbol2(self, &model->core, see); +} + +static void RescalePPMdContextVariantI(PPMdContext *self, PPMdModelVariantI *model) +{ + PPMdState *states = PPMdContextStates(self, &model->core); + int n = self->LastStateIndex + 1; + + // Bump frequency of found state + model->core.FoundState->Freq += 4; + + // Divide all frequencies and sort list + int escfreq = self->SummFreq + 4; + int adder = (model->core.OrderFall != 0 || model->MRMethod > MRM_FREEZE ? 1 : 0); + self->SummFreq = 0; + + for(int i = 0; i < n; i++) + { + escfreq -= states[i].Freq; + states[i].Freq = (states[i].Freq + adder) >> 1; + self->SummFreq += states[i].Freq; + + // Keep states sorted by decreasing frequency + if(i > 0 && states[i].Freq > states[i - 1].Freq) + { + // If not sorted, move current state upwards until list is sorted + PPMdState tmp = states[i]; + + int j = i - 1; + while(j > 0 && tmp.Freq > states[j - 1].Freq) j--; + + memmove(&states[j + 1], &states[j], sizeof(PPMdState) * (i - j)); + states[j] = tmp; + } + } + + // TODO: add better sorting stage here. + + // Drop states whose frequency has fallen to 0 + if(states[n - 1].Freq == 0) + { + int numzeros = 1; + while(numzeros < n && states[n - 1 - numzeros].Freq == 0) numzeros++; + + escfreq += numzeros; + + self->LastStateIndex -= numzeros; + if(self->LastStateIndex == 0) + { + PPMdState tmp = states[0]; + + tmp.Freq = (2 * tmp.Freq + escfreq - 1) / escfreq; + if(tmp.Freq > MAX_FREQ / 3) tmp.Freq = MAX_FREQ / 3; + + FreeUnits(model->core.alloc, self->States, (n + 1) >> 1); + model->core.FoundState = PPMdContextOneState(self); + *model->core.FoundState = tmp; + + self->Flags = (self->Flags & 0x10) + 0x08 * (tmp.Symbol >= 0x40); + + return; + } + + self->States = ShrinkUnits(model->core.alloc, self->States, (n + 1) >> 1, (self->LastStateIndex + 2) >> 1); + + PPMdState *states = PPMdContextStates(self, &model->core); + self->Flags &= ~0x08; + + for(int i = 0; i <= self->LastStateIndex; i++) + if(states[i].Symbol >= 0x40) self->Flags |= 0x08; + } + + self->SummFreq += (escfreq + 1) >> 1; + self->Flags |= 0x04; + + // The found state is the first one to breach the limit, thus it is the largest and also first + model->core.FoundState = PPMdContextStates(self, &model->core); +} diff --git a/ppmd/VariantI.h b/ppmd/VariantI.h new file mode 100644 index 0000000..78aa780 --- /dev/null +++ b/ppmd/VariantI.h @@ -0,0 +1,53 @@ +/* + * VariantI.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __PPMD_VARIANT_I_H__ +#define __PPMD_VARIANT_I_H__ + +#include "Context.h" +#include "SubAllocatorVariantI.h" + +// PPMd Variant I. Used by WinZip. + +#define MRM_RESTART 0 +#define MRM_CUT_OFF 1 +#define MRM_FREEZE 2 + +typedef struct PPMdModelVariantI +{ + PPMdCoreModel core; + + PPMdSubAllocatorVariantI *alloc; + + uint8_t NS2BSIndx[256], QTable[260]; // constants + + PPMdContext *MaxContext; + int MaxOrder, MRMethod; + SEE2Context SEE2Cont[24][32], DummySEE2Cont; + uint16_t BinSumm[25][64]; // binary SEE-contexts + + bool endofstream; +} PPMdModelVariantI; + +void StartPPMdModelVariantI(PPMdModelVariantI *self, PPMdReadFunction *readfunc, void *inputcontext, + PPMdSubAllocatorVariantI *alloc, int maxorder, int restoration); +int NextPPMdVariantIByte(PPMdModelVariantI *self); + +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53997d8..6356948 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -114,6 +114,18 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/arjz_default.bin file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/arjz_v55_new.bin DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/zip_shrink.bin + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/zip_implode.bin + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/zip_deflate64.bin + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/zip_ppmd.bin + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/) + # 'Google_Tests_run' is the target name # 'test1.cpp tests2.cpp' are source files with tests add_executable(tests_run apple_rle.cpp crc32.c crc32.h adc.cpp bzip2.cpp lzip.cpp lzfse.cpp zstd.cpp lzma.cpp flac.cpp lz4.cpp @@ -122,5 +134,6 @@ add_executable(tests_run apple_rle.cpp crc32.c crc32.h adc.cpp bzip2.cpp lzip.cp lha/lh_static.cpp lha/lh1.cpp lha/larc.cpp lha/lh_old.cpp ace/ace.cpp arj/arj.cpp - arjz/arjz.cpp) + arjz/arjz.cpp + zip/zip.cpp) target_link_libraries(tests_run gtest gtest_main "Aaru.Compression.Native") diff --git a/tests/data/zip_deflate64.bin b/tests/data/zip_deflate64.bin new file mode 100644 index 0000000..8051511 Binary files /dev/null and b/tests/data/zip_deflate64.bin differ diff --git a/tests/data/zip_implode.bin b/tests/data/zip_implode.bin new file mode 100644 index 0000000..9c63141 Binary files /dev/null and b/tests/data/zip_implode.bin differ diff --git a/tests/data/zip_ppmd.bin b/tests/data/zip_ppmd.bin new file mode 100644 index 0000000..73cfcd8 Binary files /dev/null and b/tests/data/zip_ppmd.bin differ diff --git a/tests/data/zip_shrink.bin b/tests/data/zip_shrink.bin new file mode 100644 index 0000000..95997e6 Binary files /dev/null and b/tests/data/zip_shrink.bin differ diff --git a/tests/zip/zip.cpp b/tests/zip/zip.cpp new file mode 100644 index 0000000..ec9d4b4 --- /dev/null +++ b/tests/zip/zip.cpp @@ -0,0 +1,193 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include +#include +#include +#include +#include + +#include "../../library.h" +#include "../crc32.h" +#include "gtest/gtest.h" + +/* alice29.txt: 152089 bytes, CRC32 = 0x66007dba */ +#define EXPECTED_CRC32 0x66007dba +#define EXPECTED_OUTPUT_SIZE 152089 + +/* ZIP Shrink test: PKZIP1 ES.ZIP (method 1) */ +#define SHRINK_COMPRESSED_SIZE 65014 + +static const uint8_t *shrink_buffer; + +class ZipShrinkFixture : public ::testing::Test +{ +protected: + void SetUp() + { + char path[PATH_MAX]; + char filename[PATH_MAX]; + getcwd(path, PATH_MAX); + snprintf(filename, PATH_MAX, "%s/data/zip_shrink.bin", path); + + FILE *file = fopen(filename, "rb"); + shrink_buffer = (const uint8_t *)malloc(SHRINK_COMPRESSED_SIZE); + fread((void *)shrink_buffer, 1, SHRINK_COMPRESSED_SIZE, file); + fclose(file); + } + + void TearDown() { free((void *)shrink_buffer); } +}; + +TEST_F(ZipShrinkFixture, ZipShrink) +{ + size_t destLen = EXPECTED_OUTPUT_SIZE; + auto *outBuf = (uint8_t *)malloc(EXPECTED_OUTPUT_SIZE); + + auto err = AARU_zip_shrink_decode_buffer(outBuf, &destLen, shrink_buffer, SHRINK_COMPRESSED_SIZE); + + EXPECT_EQ(err, 0); + EXPECT_EQ(destLen, EXPECTED_OUTPUT_SIZE); + + auto crc = crc32_data(outBuf, EXPECTED_OUTPUT_SIZE); + free(outBuf); + + EXPECT_EQ(crc, EXPECTED_CRC32); +} + +/* ZIP Implode test: PKZIP1 EI.ZIP (method 6, flags=0x0006: 8K dict + literals) */ +#define IMPLODE_COMPRESSED_SIZE 60488 + +static const uint8_t *implode_buffer; + +class ZipImplodeFixture : public ::testing::Test +{ +protected: + void SetUp() + { + char path[PATH_MAX]; + char filename[PATH_MAX]; + getcwd(path, PATH_MAX); + snprintf(filename, PATH_MAX, "%s/data/zip_implode.bin", path); + + FILE *file = fopen(filename, "rb"); + implode_buffer = (const uint8_t *)malloc(IMPLODE_COMPRESSED_SIZE); + fread((void *)implode_buffer, 1, IMPLODE_COMPRESSED_SIZE, file); + fclose(file); + } + + void TearDown() { free((void *)implode_buffer); } +}; + +TEST_F(ZipImplodeFixture, ZipImplode) +{ + size_t destLen = EXPECTED_OUTPUT_SIZE; + auto *outBuf = (uint8_t *)malloc(EXPECTED_OUTPUT_SIZE); + + /* large_dictionary=1 (8K), has_literals=1 (3 trees) from flags 0x0006 */ + auto err = AARU_zip_implode_decode_buffer(outBuf, &destLen, implode_buffer, IMPLODE_COMPRESSED_SIZE, 1, 1); + + EXPECT_EQ(err, 0); + EXPECT_EQ(destLen, EXPECTED_OUTPUT_SIZE); + + auto crc = crc32_data(outBuf, EXPECTED_OUTPUT_SIZE); + free(outBuf); + + EXPECT_EQ(crc, EXPECTED_CRC32); +} + +/* ZIP Deflate64 test: 7-Zip DEFLATE64 1FILE.ZIP (method 9) */ +#define DEFLATE64_COMPRESSED_SIZE 50564 + +static const uint8_t *deflate64_buffer; + +class ZipDeflate64Fixture : public ::testing::Test +{ +protected: + void SetUp() + { + char path[PATH_MAX]; + char filename[PATH_MAX]; + getcwd(path, PATH_MAX); + snprintf(filename, PATH_MAX, "%s/data/zip_deflate64.bin", path); + + FILE *file = fopen(filename, "rb"); + deflate64_buffer = (const uint8_t *)malloc(DEFLATE64_COMPRESSED_SIZE); + fread((void *)deflate64_buffer, 1, DEFLATE64_COMPRESSED_SIZE, file); + fclose(file); + } + + void TearDown() { free((void *)deflate64_buffer); } +}; + +TEST_F(ZipDeflate64Fixture, ZipDeflate64) +{ + size_t destLen = EXPECTED_OUTPUT_SIZE; + auto *outBuf = (uint8_t *)malloc(EXPECTED_OUTPUT_SIZE); + + auto err = AARU_zip_deflate64_decode_buffer(outBuf, &destLen, deflate64_buffer, DEFLATE64_COMPRESSED_SIZE); + + EXPECT_EQ(err, 0); + EXPECT_EQ(destLen, EXPECTED_OUTPUT_SIZE); + + auto crc = crc32_data(outBuf, EXPECTED_OUTPUT_SIZE); + free(outBuf); + + EXPECT_EQ(crc, EXPECTED_CRC32); +} + +/* ZIP PPMd test: 7-Zip PPMd 1FILE.ZIP (method 98, variant I) */ +/* Parameters from info word 0x0037: maxorder=8, suballocsize=4MB, restoration=0 */ +#define PPMD_COMPRESSED_SIZE 38627 + +static const uint8_t *ppmd_buffer; + +class ZipPPMdFixture : public ::testing::Test +{ +protected: + void SetUp() + { + char path[PATH_MAX]; + char filename[PATH_MAX]; + getcwd(path, PATH_MAX); + snprintf(filename, PATH_MAX, "%s/data/zip_ppmd.bin", path); + + FILE *file = fopen(filename, "rb"); + ppmd_buffer = (const uint8_t *)malloc(PPMD_COMPRESSED_SIZE); + fread((void *)ppmd_buffer, 1, PPMD_COMPRESSED_SIZE, file); + fclose(file); + } + + void TearDown() { free((void *)ppmd_buffer); } +}; + +TEST_F(ZipPPMdFixture, ZipPPMd) +{ + auto *outBuf = (uint8_t *)malloc(EXPECTED_OUTPUT_SIZE); + + /* maxorder=8, suballocsize=4194304 (4MB), restoration=0 (restart) */ + auto err = AARU_zip_ppmd_decode_buffer(outBuf, EXPECTED_OUTPUT_SIZE, ppmd_buffer, PPMD_COMPRESSED_SIZE, 8, + 4194304, 0); + + EXPECT_EQ(err, 0); + + auto crc = crc32_data(outBuf, EXPECTED_OUTPUT_SIZE); + free(outBuf); + + EXPECT_EQ(crc, EXPECTED_CRC32); +} diff --git a/wavpack/common_utils.c b/wavpack/common_utils.c new file mode 100644 index 0000000..4e432f5 --- /dev/null +++ b/wavpack/common_utils.c @@ -0,0 +1,713 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// common_utils.c + +// This module provides a lot of the trivial WavPack API functions and several +// functions that are common to both reading and writing WavPack files (like +// WavpackCloseFile()). Functions here are restricted to those that have few +// external dependancies and this is done so that applications that statically +// link to the WavPack library (like the command-line utilities on Windows) +// do not need to include the entire library image if they only use a subset +// of it. This module will be loaded for ANY WavPack application. + +#include +#include +#include + +#include "wavpack_local.h" + +#ifndef LIBWAVPACK_VERSION_STRING +#include "wavpack_version.h" +#endif + +///////////////////////////// local table storage //////////////////////////// + +const uint32_t sample_rates[] = {6000, 8000, 9600, 11025, 12000, 16000, 22050, 24000, + 32000, 44100, 48000, 64000, 88200, 96000, 192000}; + +///////////////////////////// executable code //////////////////////////////// + +// This function obtains general information about an open input file and +// returns a mask with the following bit values: + +// MODE_WVC: a .wvc file has been found and will be used for lossless +// MODE_LOSSLESS: file is lossless (either pure or hybrid) +// MODE_HYBRID: file is hybrid mode (either lossy or lossless) +// MODE_FLOAT: audio data is 32-bit ieee floating point +// MODE_VALID_TAG: file conatins a valid ID3v1 or APEv2 tag +// MODE_HIGH: file was created in "high" mode (information only) +// MODE_FAST: file was created in "fast" mode (information only) +// MODE_EXTRA: file was created using "extra" mode (information only) +// MODE_APETAG: file contains a valid APEv2 tag +// MODE_SFX: file was created as a "self-extracting" executable +// MODE_VERY_HIGH: file was created in the "very high" mode (or in +// the "high" mode prior to 4.4) +// MODE_MD5: file contains an MD5 checksum +// MODE_XMODE: level used for extra mode (1-6, 0=unknown) +// MODE_DNS: dynamic noise shaping + +int WavpackGetMode(WavpackContext *wpc) +{ + int mode = 0; + + if(wpc) + { + if(wpc->config.flags & CONFIG_HYBRID_FLAG) + mode |= MODE_HYBRID; + else if(!(wpc->config.flags & CONFIG_LOSSY_MODE)) + mode |= MODE_LOSSLESS; + + if(wpc->wvc_flag) mode |= (MODE_LOSSLESS | MODE_WVC); + + if(wpc->lossy_blocks) mode &= ~MODE_LOSSLESS; + + if(wpc->config.flags & CONFIG_FLOAT_DATA) mode |= MODE_FLOAT; + + if(wpc->config.flags & (CONFIG_HIGH_FLAG | CONFIG_VERY_HIGH_FLAG)) + { + mode |= MODE_HIGH; + + if((wpc->config.flags & CONFIG_VERY_HIGH_FLAG) || + (wpc->streams && wpc->streams[0] && wpc->streams[0]->wphdr.version < 0x405)) + mode |= MODE_VERY_HIGH; + } + + if(wpc->config.flags & CONFIG_FAST_FLAG) mode |= MODE_FAST; + + if(wpc->config.flags & CONFIG_EXTRA_MODE) mode |= (MODE_EXTRA | (wpc->config.xmode << 12)); + + if(wpc->config.flags & CONFIG_CREATE_EXE) mode |= MODE_SFX; + + if(wpc->config.flags & CONFIG_MD5_CHECKSUM) mode |= MODE_MD5; + + if((wpc->config.flags & CONFIG_HYBRID_FLAG) && (wpc->config.flags & CONFIG_DYNAMIC_SHAPING) && wpc->streams && + wpc->streams[0] && wpc->streams[0]->wphdr.version >= 0x407) + mode |= MODE_DNS; + +#ifndef NO_TAGS + if(valid_tag(&wpc->m_tag)) + { + mode |= MODE_VALID_TAG; + + if(valid_tag(&wpc->m_tag) == 'A') mode |= MODE_APETAG; + } +#endif + + mode |= (wpc->config.qmode << 16) & 0xFF0000; + } + + return mode; +} + +// This function obtains information about specific file features that were +// added for version 5.0, specifically qualifications added to support CAF +// and DSD files. Except for indicating the presence of DSD data, these +// bits are meant to simply indicate the format of the data in the original +// source file and do NOT indicate how the library will return the data to +// the appication (which is always the same). This means that in general an +// application that simply wants to play or process the audio data need not +// be concerned about these. If the file is DSD audio, then either of the +// QMDOE_DSD_LSB_FIRST or QMODE_DSD_MSB_FIRST bits will be set (but the +// DSD audio is always returned to the caller MSB first). + +// QMODE_BIG_ENDIAN 0x1 // big-endian data format (opposite of WAV format) +// QMODE_SIGNED_BYTES 0x2 // 8-bit audio data is signed (opposite of WAV format) +// QMODE_UNSIGNED_WORDS 0x4 // audio data (other than 8-bit) is unsigned (opposite of WAV format) +// QMODE_REORDERED_CHANS 0x8 // source channels were not Microsoft order, so they were reordered +// QMODE_DSD_LSB_FIRST 0x10 // DSD bytes, LSB first (most Sony .dsf files) +// QMODE_DSD_MSB_FIRST 0x20 // DSD bytes, MSB first (Philips .dff files) +// QMODE_DSD_IN_BLOCKS 0x40 // DSD data is blocked by channels (Sony .dsf only) + +int WavpackGetQualifyMode(WavpackContext *wpc) { return wpc->config.qmode & 0xFF; } + +// This function returns a pointer to a string describing the last error +// generated by WavPack. + +char *WavpackGetErrorMessage(WavpackContext *wpc) { return wpc->error_message; } + +// Get total number of samples contained in the WavPack file, or -1 if unknown + +uint32_t WavpackGetNumSamples(WavpackContext *wpc) { return (uint32_t)WavpackGetNumSamples64(wpc); } + +int64_t WavpackGetNumSamples64(WavpackContext *wpc) { return wpc ? wpc->total_samples : -1; } + +// Get the current sample index position, or -1 if unknown + +uint32_t WavpackGetSampleIndex(WavpackContext *wpc) { return (uint32_t)WavpackGetSampleIndex64(wpc); } + +int64_t WavpackGetSampleIndex64(WavpackContext *wpc) +{ + if(wpc) + { +#ifdef ENABLE_LEGACY + if(wpc->stream3) + return get_sample_index3(wpc); + else if(wpc->streams && wpc->streams[0]) + return wpc->streams[0]->sample_index; +#else + if(wpc->streams && wpc->streams[0]) return wpc->streams[0]->sample_index; +#endif + } + + return -1; +} + +// Get the number of errors encountered so far + +int WavpackGetNumErrors(WavpackContext *wpc) { return wpc ? wpc->crc_errors : 0; } + +// return TRUE if any uncorrected lossy blocks were actually written or read + +int WavpackLossyBlocks(WavpackContext *wpc) { return wpc ? wpc->lossy_blocks : 0; } + +// Calculate the progress through the file as a double from 0.0 (for begin) +// to 1.0 (for done). A return value of -1.0 indicates that the progress is +// unknown. + +double WavpackGetProgress(WavpackContext *wpc) +{ + if(wpc && wpc->total_samples != -1 && wpc->total_samples != 0) + return (double)WavpackGetSampleIndex64(wpc) / wpc->total_samples; + else + return -1.0; +} + +// Return the total size of the WavPack file(s) in bytes. + +uint32_t WavpackGetFileSize(WavpackContext *wpc) { return (uint32_t)(wpc ? wpc->filelen + wpc->file2len : 0); } + +int64_t WavpackGetFileSize64(WavpackContext *wpc) { return wpc ? wpc->filelen + wpc->file2len : 0; } + +// Calculate the ratio of the specified WavPack file size to the size of the +// original audio data as a double greater than 0.0 and (usually) smaller than +// 1.0. A value greater than 1.0 represents "negative" compression and a +// return value of 0.0 indicates that the ratio cannot be determined. + +double WavpackGetRatio(WavpackContext *wpc) +{ + if(wpc && wpc->total_samples != -1 && wpc->filelen) + { + double output_size = (double)wpc->total_samples * wpc->config.num_channels * wpc->config.bytes_per_sample; + double input_size = (double)wpc->filelen + wpc->file2len; + + if(output_size >= 1.0 && input_size >= 1.0) return input_size / output_size; + } + + return 0.0; +} + +// Calculate the average bitrate of the WavPack file in bits per second. A +// return of 0.0 indicates that the bitrate cannot be determined. An option is +// provided to use (or not use) any attendant .wvc file. + +double WavpackGetAverageBitrate(WavpackContext *wpc, int count_wvc) +{ + if(wpc && wpc->total_samples != -1 && wpc->filelen) + { + double output_time = (double)wpc->total_samples / WavpackGetSampleRate(wpc); + double input_size = (double)wpc->filelen + (count_wvc ? wpc->file2len : 0); + + if(output_time >= 0.1 && input_size >= 1.0) return input_size * 8.0 / output_time; + } + + return 0.0; +} + +// Calculate the bitrate of the current WavPack file block in bits per second. +// This can be used for an "instant" bit display and gets updated from about +// 1 to 4 times per second. A return of 0.0 indicates that the bitrate cannot +// be determined. + +double WavpackGetInstantBitrate(WavpackContext *wpc) +{ + if(wpc && wpc->stream3) return WavpackGetAverageBitrate(wpc, TRUE); + + if(wpc && wpc->streams && wpc->streams[0] && wpc->streams[0]->wphdr.block_samples) + { + double output_time = (double)wpc->streams[0]->wphdr.block_samples / WavpackGetSampleRate(wpc); + double input_size = 0; + int si; + + for(si = 0; si < wpc->num_streams; ++si) + { + if(wpc->streams[si]->blockbuff) input_size += ((WavpackHeader *)wpc->streams[si]->blockbuff)->ckSize; + + if(wpc->streams[si]->block2buff) input_size += ((WavpackHeader *)wpc->streams[si]->block2buff)->ckSize; + } + + if(output_time > 0.0 && input_size >= 1.0) return input_size * 8.0 / output_time; + } + + return 0.0; +} + +// This function allows retrieving the Core Audio File channel layout, many of which do not +// conform to the Microsoft ordering standard that WavPack requires internally (at least for +// those channels present in the "channel mask"). In addition to the layout tag, this function +// returns the reordering string (if stored in the file) to allow the unpacker to reorder the +// channels back to the specified layout (if it wants to restore the CAF order). The number of +// channels in the layout is determined from the lower nybble of the layout word (and should +// probably match the number of channels in the file), and if a reorder string is requested +// then that much space must be allocated. Note that all the reordering is actually done +// outside of this library, and that if reordering is done then the appropriate qmode bit +// will be set. +// +// Note: Normally this function would not be used by an application unless it specifically +// wanted to restore a non-standard channel order (to check an MD5, for example) or obtain +// the Core Audio channel layout ID. For simple file decoding for playback, the channel_mask +// should provide all the information required unless there are non-Microsoft channels +// involved, in which case WavpackGetChannelIdentities() will provide the identities of +// the other channels (if they are known). + +uint32_t WavpackGetChannelLayout(WavpackContext *wpc, unsigned char *reorder) +{ + if((wpc->channel_layout & 0xff) && wpc->channel_reordering && reorder) + memcpy(reorder, wpc->channel_reordering, wpc->channel_layout & 0xff); + + return wpc->channel_layout; +} + +// This function provides the identities of ALL the channels in the file, including the +// standard Microsoft channels (which come first, in order, and are numbered 1-18) and also +// any non-Microsoft channels (which can be in any order and have values from 33-254). The +// value 0x00 is invalid and 0xFF indicates an "unknown" or "unnassigned" channel. The +// string is NULL terminated so the caller must supply enough space for the number +// of channels indicated by WavpackGetNumChannels(), plus one. +// +// Note that this function returns the actual order of the channels in the Wavpack file +// (i.e., the order returned by WavpackUnpackSamples()). If the file includes a "reordering" +// string because the source file was not in Microsoft order that is NOT taken into account +// here and really only needs to be considered if doing an MD5 verification or if it's +// required to restore the original order/file (like wvunpack does). + +void WavpackGetChannelIdentities(WavpackContext *wpc, unsigned char *identities) +{ + int num_channels = wpc->config.num_channels, index = 1; + uint32_t channel_mask = wpc->config.channel_mask; + unsigned char *src = wpc->channel_identities; + + while(num_channels--) + { + if(channel_mask) + { + while(!(channel_mask & 1)) + { + channel_mask >>= 1; + index++; + } + + *identities++ = index++; + channel_mask >>= 1; + } + else if(src && *src) + *identities++ = *src++; + else + *identities++ = 0xff; + } + + *identities = 0; +} + +// Close the specified WavPack file and release all resources used by it. +// Returns NULL. + +WavpackContext *WavpackCloseFile(WavpackContext *wpc) +{ + if(wpc->streams) + { + free_streams(wpc); + + if(wpc->streams[0]) free(wpc->streams[0]); + + free(wpc->streams); + } + +#ifdef ENABLE_LEGACY + if(wpc->stream3) free_stream3(wpc); +#endif + + if(wpc->reader && wpc->reader->close && wpc->wv_in) wpc->reader->close(wpc->wv_in); + + if(wpc->reader && wpc->reader->close && wpc->wvc_in) wpc->reader->close(wpc->wvc_in); + + WavpackFreeWrapper(wpc); + + if(wpc->channel_reordering) free(wpc->channel_reordering); + +#ifndef NO_TAGS + free_tag(&wpc->m_tag); +#endif + +#ifdef ENABLE_DSD + if(wpc->decimation_context) decimate_dsd_destroy(wpc->decimation_context); +#endif + + free(wpc); + + return NULL; +} + +// These routines are used to access (and free) header and trailer data that +// was retrieved from the Wavpack file. The header will be available before +// the samples are decoded and the trailer will be available after all samples +// have been read. + +uint32_t WavpackGetWrapperBytes(WavpackContext *wpc) { return wpc ? wpc->wrapper_bytes : 0; } + +unsigned char *WavpackGetWrapperData(WavpackContext *wpc) { return wpc ? wpc->wrapper_data : NULL; } + +void WavpackFreeWrapper(WavpackContext *wpc) +{ + if(wpc && wpc->wrapper_data) + { + free(wpc->wrapper_data); + wpc->wrapper_data = NULL; + wpc->wrapper_bytes = 0; + } +} + +// Returns the sample rate of the specified WavPack file + +uint32_t WavpackGetSampleRate(WavpackContext *wpc) +{ + return wpc ? (wpc->dsd_multiplier ? wpc->config.sample_rate * wpc->dsd_multiplier : wpc->config.sample_rate) + : 44100; +} + +// Returns the native sample rate of the specified WavPack file +// (provides the native rate for DSD files rather than the "byte" rate that's used for +// seeking, duration, etc. and would generally be used just for user facing reports) + +uint32_t WavpackGetNativeSampleRate(WavpackContext *wpc) +{ + return wpc ? (wpc->dsd_multiplier ? wpc->config.sample_rate * wpc->dsd_multiplier * 8 : wpc->config.sample_rate) + : 44100; +} + +// Returns the number of channels of the specified WavPack file. Note that +// this is the actual number of channels contained in the file even if the +// OPEN_2CH_MAX flag was specified when the file was opened. + +int WavpackGetNumChannels(WavpackContext *wpc) { return wpc ? wpc->config.num_channels : 2; } + +// Returns the standard Microsoft channel mask for the specified WavPack +// file. A value of zero indicates that there is no speaker assignment +// information. + +int WavpackGetChannelMask(WavpackContext *wpc) { return wpc ? wpc->config.channel_mask : 0; } + +// Return the normalization value for floating point data (valid only +// if floating point data is present). A value of 127 indicates that +// the floating point range is +/- 1.0. Higher values indicate a +// larger floating point range. + +int WavpackGetFloatNormExp(WavpackContext *wpc) { return wpc->config.float_norm_exp; } + +// Returns the actual number of valid bits per sample contained in the +// original file, which may or may not be a multiple of 8. Floating data +// always has 32 bits, integers may be from 1 to 32 bits each. When this +// value is not a multiple of 8, then the "extra" bits are located in the +// LSBs of the results. That is, values are right justified when unpacked +// into ints, but are left justified in the number of bytes used by the +// original data. + +int WavpackGetBitsPerSample(WavpackContext *wpc) { return wpc ? wpc->config.bits_per_sample : 16; } + +// Returns the number of bytes used for each sample (1 to 4) in the original +// file. This is required information for the user of this module because the +// audio data is returned in the LOWER bytes of the long buffer and must be +// left-shifted 8, 16, or 24 bits if normalized longs are required. + +int WavpackGetBytesPerSample(WavpackContext *wpc) { return wpc ? wpc->config.bytes_per_sample : 2; } + +// If the OPEN_2CH_MAX flag is specified when opening the file, this function +// will return the actual number of channels decoded from the file (which may +// or may not be less than the actual number of channels, but will always be +// 1 or 2). Normally, this will be the front left and right channels of a +// multichannel file. + +int WavpackGetReducedChannels(WavpackContext *wpc) +{ + if(wpc) + return wpc->reduced_channels ? wpc->reduced_channels : wpc->config.num_channels; + else + return 2; +} + +// Free all memory allocated for raw WavPack blocks (for all allocated streams) +// and free all additonal streams. This does not free the default stream ([0]) +// which is always kept around. + +void free_streams(WavpackContext *wpc) +{ + int si = wpc->num_streams; + + while(si--) + { + if(wpc->streams[si]->blockbuff) + { + free(wpc->streams[si]->blockbuff); + wpc->streams[si]->blockbuff = NULL; + } + + if(wpc->streams[si]->block2buff) + { + free(wpc->streams[si]->block2buff); + wpc->streams[si]->block2buff = NULL; + } + + if(wpc->streams[si]->sample_buffer) + { + free(wpc->streams[si]->sample_buffer); + wpc->streams[si]->sample_buffer = NULL; + } + + if(wpc->streams[si]->dc.shaping_data) + { + free(wpc->streams[si]->dc.shaping_data); + wpc->streams[si]->dc.shaping_data = NULL; + } + +#ifdef ENABLE_DSD + if(wpc->streams[si]->dsd.probabilities) + { + free(wpc->streams[si]->dsd.probabilities); + wpc->streams[si]->dsd.probabilities = NULL; + } + + if(wpc->streams[si]->dsd.summed_probabilities) + { + free(wpc->streams[si]->dsd.summed_probabilities); + wpc->streams[si]->dsd.summed_probabilities = NULL; + } + + if(wpc->streams[si]->dsd.value_lookup) + { + int i; + + for(i = 0; i < wpc->streams[si]->dsd.history_bins; ++i) + if(wpc->streams[si]->dsd.value_lookup[i]) free(wpc->streams[si]->dsd.value_lookup[i]); + + free(wpc->streams[si]->dsd.value_lookup); + wpc->streams[si]->dsd.value_lookup = NULL; + } + + if(wpc->streams[si]->dsd.ptable) + { + free(wpc->streams[si]->dsd.ptable); + wpc->streams[si]->dsd.ptable = NULL; + } +#endif + + if(si) + { + wpc->num_streams--; + free(wpc->streams[si]); + wpc->streams[si] = NULL; + } + } + + wpc->current_stream = 0; +} + +void WavpackFloatNormalize(int32_t *values, int32_t num_values, int delta_exp) +{ + f32 *fvalues = (f32 *)values; + int exp; + + if(!delta_exp) return; + + while(num_values--) + { + if((exp = get_exponent(*fvalues)) == 0 || exp + delta_exp <= 0) + *fvalues = 0; + else if(exp == 255 || (exp += delta_exp) >= 255) + { + set_exponent(*fvalues, 255); + set_mantissa(*fvalues, 0); + } + else + set_exponent(*fvalues, exp); + + fvalues++; + } +} + +void WavpackLittleEndianToNative(void *data, char *format) +{ + unsigned char *cp = (unsigned char *)data; + int64_t temp; + + while(*format) + { + switch(*format) + { + case 'D': + temp = cp[0] + ((int64_t)cp[1] << 8) + ((int64_t)cp[2] << 16) + ((int64_t)cp[3] << 24) + + ((int64_t)cp[4] << 32) + ((int64_t)cp[5] << 40) + ((int64_t)cp[6] << 48) + + ((int64_t)cp[7] << 56); + *(int64_t *)cp = temp; + cp += 8; + break; + + case 'L': + temp = cp[0] + ((int32_t)cp[1] << 8) + ((int32_t)cp[2] << 16) + ((int32_t)cp[3] << 24); + *(int32_t *)cp = (int32_t)temp; + cp += 4; + break; + + case 'S': + temp = cp[0] + (cp[1] << 8); + *(int16_t *)cp = (int16_t)temp; + cp += 2; + break; + + default: + if(isdigit(*format)) cp += *format - '0'; + + break; + } + + format++; + } +} + +void WavpackNativeToLittleEndian(void *data, char *format) +{ + unsigned char *cp = (unsigned char *)data; + int64_t temp; + + while(*format) + { + switch(*format) + { + case 'D': + temp = *(int64_t *)cp; + *cp++ = (unsigned char)temp; + *cp++ = (unsigned char)(temp >> 8); + *cp++ = (unsigned char)(temp >> 16); + *cp++ = (unsigned char)(temp >> 24); + *cp++ = (unsigned char)(temp >> 32); + *cp++ = (unsigned char)(temp >> 40); + *cp++ = (unsigned char)(temp >> 48); + *cp++ = (unsigned char)(temp >> 56); + break; + + case 'L': + temp = *(int32_t *)cp; + *cp++ = (unsigned char)temp; + *cp++ = (unsigned char)(temp >> 8); + *cp++ = (unsigned char)(temp >> 16); + *cp++ = (unsigned char)(temp >> 24); + break; + + case 'S': + temp = *(int16_t *)cp; + *cp++ = (unsigned char)temp; + *cp++ = (unsigned char)(temp >> 8); + break; + + default: + if(isdigit(*format)) cp += *format - '0'; + + break; + } + + format++; + } +} + +void WavpackBigEndianToNative(void *data, char *format) +{ + unsigned char *cp = (unsigned char *)data; + int64_t temp; + + while(*format) + { + switch(*format) + { + case 'D': + temp = cp[7] + ((int64_t)cp[6] << 8) + ((int64_t)cp[5] << 16) + ((int64_t)cp[4] << 24) + + ((int64_t)cp[3] << 32) + ((int64_t)cp[2] << 40) + ((int64_t)cp[1] << 48) + + ((int64_t)cp[0] << 56); + *(int64_t *)cp = temp; + cp += 8; + break; + + case 'L': + temp = cp[3] + ((int32_t)cp[2] << 8) + ((int32_t)cp[1] << 16) + ((int32_t)cp[0] << 24); + *(int32_t *)cp = (int32_t)temp; + cp += 4; + break; + + case 'S': + temp = cp[1] + (cp[0] << 8); + *(int16_t *)cp = (int16_t)temp; + cp += 2; + break; + + default: + if(isdigit(*format)) cp += *format - '0'; + + break; + } + + format++; + } +} + +void WavpackNativeToBigEndian(void *data, char *format) +{ + unsigned char *cp = (unsigned char *)data; + int64_t temp; + + while(*format) + { + switch(*format) + { + case 'D': + temp = *(int64_t *)cp; + *cp++ = (unsigned char)(temp >> 56); + *cp++ = (unsigned char)(temp >> 48); + *cp++ = (unsigned char)(temp >> 40); + *cp++ = (unsigned char)(temp >> 32); + *cp++ = (unsigned char)(temp >> 24); + *cp++ = (unsigned char)(temp >> 16); + *cp++ = (unsigned char)(temp >> 8); + *cp++ = (unsigned char)temp; + break; + + case 'L': + temp = *(int32_t *)cp; + *cp++ = (unsigned char)(temp >> 24); + *cp++ = (unsigned char)(temp >> 16); + *cp++ = (unsigned char)(temp >> 8); + *cp++ = (unsigned char)temp; + break; + + case 'S': + temp = *(int16_t *)cp; + *cp++ = (unsigned char)(temp >> 8); + *cp++ = (unsigned char)temp; + break; + + default: + if(isdigit(*format)) cp += *format - '0'; + + break; + } + + format++; + } +} + +uint32_t WavpackGetLibraryVersion(void) +{ return (LIBWAVPACK_MAJOR << 16) | (LIBWAVPACK_MINOR << 8) | (LIBWAVPACK_MICRO << 0); } + +const char *WavpackGetLibraryVersionString(void) { return LIBWAVPACK_VERSION_STRING; } diff --git a/wavpack/decorr_utils.c b/wavpack/decorr_utils.c new file mode 100644 index 0000000..db6c0b3 --- /dev/null +++ b/wavpack/decorr_utils.c @@ -0,0 +1,210 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// decorr_utils.c + +// This module contains the functions that process metadata blocks that are +// specific to the decorrelator. These would be called any time a WavPack +// block was parsed. These are in a module separate from the actual unpack +// decorrelation code (unpack.c) so that if an application just wants to get +// information from WavPack files (rather than actually decoding audio) then +// less code needs to be linked. + +#include +#include + +#include "wavpack_local.h" + +///////////////////////////// executable code //////////////////////////////// + +// Read decorrelation terms from specified metadata block into the +// decorr_passes array. The terms range from -3 to 8, plus 17 & 18; +// other values are reserved and generate errors for now. The delta +// ranges from 0 to 7 with all values valid. Note that the terms are +// stored in the opposite order in the decorr_passes array compared +// to packing. + +int read_decorr_terms(WavpackStream *wps, WavpackMetadata *wpmd) +{ + int termcnt = wpmd->byte_length; + unsigned char *byteptr = wpmd->data; + struct decorr_pass *dpp; + + if(termcnt > MAX_NTERMS) return FALSE; + + wps->num_terms = termcnt; + + for(dpp = wps->decorr_passes + termcnt - 1; termcnt--; dpp--) + { + dpp->term = (int)(*byteptr & 0x1f) - 5; + dpp->delta = (*byteptr++ >> 5) & 0x7; + + if(!dpp->term || dpp->term < -3 || (dpp->term > MAX_TERM && dpp->term < 17) || dpp->term > 18 || + ((wps->wphdr.flags & MONO_DATA) && dpp->term < 0)) + return FALSE; + } + + return TRUE; +} + +// Read decorrelation weights from specified metadata block into the +// decorr_passes array. The weights range +/-1024, but are rounded and +// truncated to fit in signed chars for metadata storage. Weights are +// separate for the two channels and are specified from the "last" term +// (first during encode). Unspecified weights are set to zero. + +int read_decorr_weights(WavpackStream *wps, WavpackMetadata *wpmd) +{ + int termcnt = wpmd->byte_length, tcount; + char *byteptr = wpmd->data; + struct decorr_pass *dpp; + + if(!(wps->wphdr.flags & MONO_DATA)) termcnt /= 2; + + if(termcnt > wps->num_terms) return FALSE; + + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) dpp->weight_A = dpp->weight_B = 0; + + while(--dpp >= wps->decorr_passes && termcnt--) + { + dpp->weight_A = restore_weight(*byteptr++); + + if(!(wps->wphdr.flags & MONO_DATA)) dpp->weight_B = restore_weight(*byteptr++); + } + + return TRUE; +} + +// Read decorrelation samples from specified metadata block into the +// decorr_passes array. The samples are signed 32-bit values, but are +// converted to signed log2 values for storage in metadata. Values are +// stored for both channels and are specified from the "last" term +// (first during encode) with unspecified samples set to zero. The +// number of samples stored varies with the actual term value, so +// those must obviously come first in the metadata. + +int read_decorr_samples(WavpackStream *wps, WavpackMetadata *wpmd) +{ + unsigned char *byteptr = wpmd->data; + unsigned char *endptr = byteptr + wpmd->byte_length; + struct decorr_pass *dpp; + int tcount; + + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + CLEAR(dpp->samples_A); + CLEAR(dpp->samples_B); + } + + if(wps->wphdr.version == 0x402 && (wps->wphdr.flags & HYBRID_FLAG)) + { + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 2 : 4) > endptr) return FALSE; + + wps->dc.error[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->dc.error[1] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + } + } + + while(dpp-- > wps->decorr_passes && byteptr < endptr) + if(dpp->term > MAX_TERM) + { + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 4 : 8) > endptr) return FALSE; + + dpp->samples_A[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + dpp->samples_A[1] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + byteptr += 4; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + dpp->samples_B[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + dpp->samples_B[1] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + byteptr += 4; + } + } + else if(dpp->term < 0) + { + if(byteptr + 4 > endptr) return FALSE; + + dpp->samples_A[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + dpp->samples_B[0] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + byteptr += 4; + } + else + { + int m = 0, cnt = dpp->term; + + while(cnt--) + { + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 2 : 4) > endptr) return FALSE; + + dpp->samples_A[m] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + dpp->samples_B[m] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + } + + m++; + } + } + + return byteptr == endptr; +} + +// Read the shaping weights from specified metadata block into the +// WavpackStream structure. Note that there must be two values (even +// for mono streams) and that the values are stored in the same +// manner as decorrelation weights. These would normally be read from +// the "correction" file and are used for lossless reconstruction of +// hybrid data. + +int read_shaping_info(WavpackStream *wps, WavpackMetadata *wpmd) +{ + if(wpmd->byte_length == 2) + { + char *byteptr = wpmd->data; + + wps->dc.shaping_acc[0] = (int32_t)restore_weight(*byteptr++) << 16; + wps->dc.shaping_acc[1] = (int32_t)restore_weight(*byteptr++) << 16; + return TRUE; + } + else if(wpmd->byte_length >= (wps->wphdr.flags & MONO_DATA ? 4 : 8)) + { + unsigned char *byteptr = wpmd->data; + + wps->dc.error[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + wps->dc.shaping_acc[0] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + byteptr += 4; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->dc.error[1] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + wps->dc.shaping_acc[1] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + byteptr += 4; + } + + if(wpmd->byte_length == (wps->wphdr.flags & MONO_DATA ? 6 : 12)) + { + wps->dc.shaping_delta[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + + if(!(wps->wphdr.flags & MONO_DATA)) + wps->dc.shaping_delta[1] = wp_exp2s((int16_t)(byteptr[2] + (byteptr[3] << 8))); + } + + return TRUE; + } + + return FALSE; +} diff --git a/wavpack/entropy_utils.c b/wavpack/entropy_utils.c new file mode 100644 index 0000000..7dd5a5a --- /dev/null +++ b/wavpack/entropy_utils.c @@ -0,0 +1,369 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// entropy_utils.c + +// This module contains the functions that process metadata blocks that are +// specific to the entropy decoder; these would be called any time a WavPack +// block was parsed. Additionally, it contains tables and functions that are +// common to both entropy coding and decoding. These are in a module separate +// from the actual entropy encoder (write_words.c) and decoder (read_words.c) +// so that if applications that just do a subset of the full WavPack reading +// and writing can link with a subset of the library. + +#include +#include + +#include "wavpack_local.h" + +///////////////////////////// local table storage //////////////////////////// + +const uint32_t bitset[] = {1L << 0, 1L << 1, 1L << 2, 1L << 3, 1L << 4, 1L << 5, 1L << 6, 1L << 7, + 1L << 8, 1L << 9, 1L << 10, 1L << 11, 1L << 12, 1L << 13, 1L << 14, 1L << 15, + 1L << 16, 1L << 17, 1L << 18, 1L << 19, 1L << 20, 1L << 21, 1L << 22, 1L << 23, + 1L << 24, 1L << 25, 1L << 26, 1L << 27, 1L << 28, 1L << 29, 1L << 30, 1L << 31}; + +const uint32_t bitmask[] = { + (1L << 0) - 1, (1L << 1) - 1, (1L << 2) - 1, (1L << 3) - 1, (1L << 4) - 1, (1L << 5) - 1, (1L << 6) - 1, + (1L << 7) - 1, (1L << 8) - 1, (1L << 9) - 1, (1L << 10) - 1, (1L << 11) - 1, (1L << 12) - 1, (1L << 13) - 1, + (1L << 14) - 1, (1L << 15) - 1, (1L << 16) - 1, (1L << 17) - 1, (1L << 18) - 1, (1L << 19) - 1, (1L << 20) - 1, + (1L << 21) - 1, (1L << 22) - 1, (1L << 23) - 1, (1L << 24) - 1, (1L << 25) - 1, (1L << 26) - 1, (1L << 27) - 1, + (1L << 28) - 1, (1L << 29) - 1, (1L << 30) - 1, 0x7fffffff}; + +const char nbits_table[] = { + 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, // 0 - 15 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, // 16 - 31 + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 32 - 47 + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, // 48 - 63 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 64 - 79 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 80 - 95 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 96 - 111 + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 112 - 127 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 128 - 143 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 144 - 159 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 160 - 175 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 176 - 191 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 192 - 207 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 208 - 223 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, // 224 - 239 + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 // 240 - 255 +}; + +static const unsigned char log2_table[] = { + 0x00, 0x01, 0x03, 0x04, 0x06, 0x07, 0x09, 0x0a, 0x0b, 0x0d, 0x0e, 0x10, 0x11, 0x12, 0x14, 0x15, 0x16, 0x18, 0x19, + 0x1a, 0x1c, 0x1d, 0x1e, 0x20, 0x21, 0x22, 0x24, 0x25, 0x26, 0x28, 0x29, 0x2a, 0x2c, 0x2d, 0x2e, 0x2f, 0x31, 0x32, + 0x33, 0x34, 0x36, 0x37, 0x38, 0x39, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x41, 0x42, 0x43, 0x44, 0x45, 0x47, 0x48, 0x49, + 0x4a, 0x4b, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, + 0x9b, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xa9, 0xaa, 0xab, + 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xb9, 0xba, 0xbb, 0xbc, + 0xbd, 0xbe, 0xbf, 0xc0, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, + 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe4, 0xe5, 0xe6, 0xe7, 0xe7, 0xe8, 0xe9, 0xea, 0xea, + 0xeb, 0xec, 0xed, 0xee, 0xee, 0xef, 0xf0, 0xf1, 0xf1, 0xf2, 0xf3, 0xf4, 0xf4, 0xf5, 0xf6, 0xf7, 0xf7, 0xf8, 0xf9, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfc, 0xfd, 0xfe, 0xff, 0xff}; + +static const unsigned char exp2_table[] = { + 0x00, 0x01, 0x01, 0x02, 0x03, 0x03, 0x04, 0x05, 0x06, 0x06, 0x07, 0x08, 0x08, 0x09, 0x0a, 0x0b, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0e, 0x0f, 0x10, 0x10, 0x11, 0x12, 0x13, 0x13, 0x14, 0x15, 0x16, 0x16, 0x17, 0x18, 0x19, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1d, 0x1e, 0x1f, 0x20, 0x20, 0x21, 0x22, 0x23, 0x24, 0x24, 0x25, 0x26, 0x27, 0x28, 0x28, 0x29, 0x2a, + 0x2b, 0x2c, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, + 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x48, 0x49, 0x4a, + 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, + 0x5d, 0x5e, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, + 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, + 0x82, 0x83, 0x84, 0x85, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa8, 0xa9, 0xaa, 0xab, + 0xac, 0xad, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc2, + 0xc3, 0xc4, 0xc5, 0xc6, 0xc8, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf, 0xd0, 0xd2, 0xd3, 0xd4, 0xd6, 0xd7, 0xd8, 0xd9, + 0xdb, 0xdc, 0xdd, 0xde, 0xe0, 0xe1, 0xe2, 0xe4, 0xe5, 0xe6, 0xe8, 0xe9, 0xea, 0xec, 0xed, 0xee, 0xf0, 0xf1, 0xf2, + 0xf4, 0xf5, 0xf6, 0xf8, 0xf9, 0xfa, 0xfc, 0xfd, 0xff}; + +///////////////////////////// executable code //////////////////////////////// + +// Read the median log2 values from the specifed metadata structure, convert +// them back to 32-bit unsigned values and store them. If length is not +// exactly correct then we flag and return an error. + +int read_entropy_vars(WavpackStream *wps, WavpackMetadata *wpmd) +{ + unsigned char *byteptr = wpmd->data; + + if(wpmd->byte_length != ((wps->wphdr.flags & MONO_DATA) ? 6 : 12)) return FALSE; + + wps->w.c[0].median[0] = wp_exp2s(byteptr[0] + (byteptr[1] << 8)); + wps->w.c[0].median[1] = wp_exp2s(byteptr[2] + (byteptr[3] << 8)); + wps->w.c[0].median[2] = wp_exp2s(byteptr[4] + (byteptr[5] << 8)); + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->w.c[1].median[0] = wp_exp2s(byteptr[6] + (byteptr[7] << 8)); + wps->w.c[1].median[1] = wp_exp2s(byteptr[8] + (byteptr[9] << 8)); + wps->w.c[1].median[2] = wp_exp2s(byteptr[10] + (byteptr[11] << 8)); + } + + return TRUE; +} + +// Read the hybrid related values from the specifed metadata structure, convert +// them back to their internal formats and store them. The extended profile +// stuff is not implemented yet, so return an error if we get more data than +// we know what to do with. + +int read_hybrid_profile(WavpackStream *wps, WavpackMetadata *wpmd) +{ + unsigned char *byteptr = wpmd->data; + unsigned char *endptr = byteptr + wpmd->byte_length; + + if(wps->wphdr.flags & HYBRID_BITRATE) + { + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 2 : 4) > endptr) return FALSE; + + wps->w.c[0].slow_level = wp_exp2s(byteptr[0] + (byteptr[1] << 8)); + byteptr += 2; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->w.c[1].slow_level = wp_exp2s(byteptr[0] + (byteptr[1] << 8)); + byteptr += 2; + } + } + + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 2 : 4) > endptr) return FALSE; + + wps->w.bitrate_acc[0] = (int32_t)(byteptr[0] + (byteptr[1] << 8)) << 16; + byteptr += 2; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->w.bitrate_acc[1] = (int32_t)(byteptr[0] + (byteptr[1] << 8)) << 16; + byteptr += 2; + } + + if(byteptr < endptr) + { + if(byteptr + (wps->wphdr.flags & MONO_DATA ? 2 : 4) > endptr) return FALSE; + + wps->w.bitrate_delta[0] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + + if(!(wps->wphdr.flags & MONO_DATA)) + { + wps->w.bitrate_delta[1] = wp_exp2s((int16_t)(byteptr[0] + (byteptr[1] << 8))); + byteptr += 2; + } + + if(byteptr < endptr) return FALSE; + } + else + wps->w.bitrate_delta[0] = wps->w.bitrate_delta[1] = 0; + + return TRUE; +} + +// This function is called during both encoding and decoding of hybrid data to +// update the "error_limit" variable which determines the maximum sample error +// allowed in the main bitstream. In the HYBRID_BITRATE mode (which is the only +// currently implemented) this is calculated from the slow_level values and the +// bitrate accumulators. Note that the bitrate accumulators can be changing. + +void update_error_limit(WavpackStream *wps) +{ + int bitrate_0 = (wps->w.bitrate_acc[0] += wps->w.bitrate_delta[0]) >> 16; + + if(wps->wphdr.flags & MONO_DATA) + { + if(wps->wphdr.flags & HYBRID_BITRATE) + { + int slow_log_0 = (wps->w.c[0].slow_level + SLO) >> SLS; + + if(slow_log_0 - bitrate_0 > -0x100) + wps->w.c[0].error_limit = wp_exp2s(slow_log_0 - bitrate_0 + 0x100); + else + wps->w.c[0].error_limit = 0; + } + else + wps->w.c[0].error_limit = wp_exp2s(bitrate_0); + } + else + { + int bitrate_1 = (wps->w.bitrate_acc[1] += wps->w.bitrate_delta[1]) >> 16; + + if(wps->wphdr.flags & HYBRID_BITRATE) + { + int slow_log_0 = (wps->w.c[0].slow_level + SLO) >> SLS; + int slow_log_1 = (wps->w.c[1].slow_level + SLO) >> SLS; + + if(wps->wphdr.flags & HYBRID_BALANCE) + { + int balance = (slow_log_1 - slow_log_0 + bitrate_1 + 1) >> 1; + + if(balance > bitrate_0) + { + bitrate_1 = bitrate_0 * 2; + bitrate_0 = 0; + } + else if(-balance > bitrate_0) + { + bitrate_0 = bitrate_0 * 2; + bitrate_1 = 0; + } + else + { + bitrate_1 = bitrate_0 + balance; + bitrate_0 = bitrate_0 - balance; + } + } + + if(slow_log_0 - bitrate_0 > -0x100) + wps->w.c[0].error_limit = wp_exp2s(slow_log_0 - bitrate_0 + 0x100); + else + wps->w.c[0].error_limit = 0; + + if(slow_log_1 - bitrate_1 > -0x100) + wps->w.c[1].error_limit = wp_exp2s(slow_log_1 - bitrate_1 + 0x100); + else + wps->w.c[1].error_limit = 0; + } + else + { + wps->w.c[0].error_limit = wp_exp2s(bitrate_0); + wps->w.c[1].error_limit = wp_exp2s(bitrate_1); + } + } +} + +// The concept of a base 2 logarithm is used in many parts of WavPack. It is +// a way of sufficiently accurately representing 32-bit signed and unsigned +// values storing only 16 bits (actually fewer). It is also used in the hybrid +// mode for quickly comparing the relative magnitude of large values (i.e. +// division) and providing smooth exponentials using only addition. + +// These are not strict logarithms in that they become linear around zero and +// can therefore represent both zero and negative values. They have 8 bits +// of precision and in "roundtrip" conversions the total error never exceeds 1 +// part in 225 except for the cases of +/-115 and +/-195 (which error by 1). + +// This function returns the log2 for the specified 32-bit unsigned value. +// The maximum value allowed is about 0xff800000 and returns 8447. + +int FASTCALL wp_log2(uint32_t avalue) +{ + int dbits; + + if((avalue += avalue >> 9) < (1 << 8)) + { + dbits = nbits_table[avalue]; + return (dbits << 8) + log2_table[(avalue << (9 - dbits)) & 0xff]; + } + else + { + if(avalue < (1L << 16)) + dbits = nbits_table[avalue >> 8] + 8; + else if(avalue < (1L << 24)) + dbits = nbits_table[avalue >> 16] + 16; + else + dbits = nbits_table[avalue >> 24] + 24; + + return (dbits << 8) + log2_table[(avalue >> (dbits - 9)) & 0xff]; + } +} + +// This function scans a buffer of longs and accumulates the total log2 value +// of all the samples. This is useful for determining maximum compression +// because the bitstream storage required for entropy coding is proportional +// to the base 2 log of the samples. On some platforms there is an assembly +// version of this. + +#if !defined(OPT_ASM_X86) && !defined(OPT_ASM_X64) + +uint32_t log2buffer(int32_t *samples, uint32_t num_samples, int limit) +{ + uint32_t result = 0, avalue; + int dbits; + + while(num_samples--) + { + avalue = abs(*samples++); + + if((avalue += avalue >> 9) < (1 << 8)) + { + dbits = nbits_table[avalue]; + result += (dbits << 8) + log2_table[(avalue << (9 - dbits)) & 0xff]; + } + else + { + if(avalue < (1L << 16)) + dbits = nbits_table[avalue >> 8] + 8; + else if(avalue < (1L << 24)) + dbits = nbits_table[avalue >> 16] + 16; + else + dbits = nbits_table[avalue >> 24] + 24; + + result += dbits = (dbits << 8) + log2_table[(avalue >> (dbits - 9)) & 0xff]; + + if(limit && dbits >= limit) return (uint32_t)-1; + } + } + + return result; +} + +#endif + +// This function returns the log2 for the specified 32-bit signed value. +// All input values are valid and the return values are in the range of +// +/- 8192. + +int wp_log2s(int32_t value) { return (value < 0) ? -wp_log2(-value) : wp_log2(value); } + +// This function returns the original integer represented by the supplied +// logarithm (at least within the provided accuracy). The log is signed, +// but since a full 32-bit value is returned this can be used for unsigned +// conversions as well (i.e. the input range is -8192 to +8447). + +int32_t wp_exp2s(int log) +{ + uint32_t value; + + if(log < 0) return -wp_exp2s(-log); + + value = exp2_table[log & 0xff] | 0x100; + + if((log >>= 8) <= 9) + return value >> (9 - log); + else + return value << (log - 9); +} + +// These two functions convert internal weights (which are normally +/-1024) +// to and from an 8-bit signed character version for storage in metadata. The +// weights are clipped here in the case that they are outside that range. + +signed char store_weight(int weight) +{ + if(weight > 1024) + weight = 1024; + else if(weight < -1024) + weight = -1024; + + if(weight > 0) weight -= (weight + 64) >> 7; + + return (weight + 4) >> 3; +} + +int restore_weight(signed char weight) +{ + int result; + + if((result = (int)weight << 3) > 0) result += (result + 64) >> 7; + + return result; +} diff --git a/wavpack/open_legacy.c b/wavpack/open_legacy.c new file mode 100644 index 0000000..937a5b2 --- /dev/null +++ b/wavpack/open_legacy.c @@ -0,0 +1,116 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2016 David Bryant. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// open_legacy.c + +// This code provides an interface between the new reader callback mechanism that +// WavPack uses internally and the old reader callback functions that did not +// provide large file support. + +#include +#include + +#include "wavpack_local.h" + +typedef struct +{ + WavpackStreamReader *reader; + void *id; +} WavpackReaderTranslator; + +static int32_t trans_read_bytes(void *id, void *data, int32_t bcount) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->read_bytes(trans->id, data, bcount); +} + +static int32_t trans_write_bytes(void *id, void *data, int32_t bcount) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->write_bytes(trans->id, data, bcount); +} + +static int64_t trans_get_pos(void *id) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->get_pos(trans->id); +} + +static int trans_set_pos_abs(void *id, int64_t pos) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->set_pos_abs(trans->id, (uint32_t)pos); +} + +static int trans_set_pos_rel(void *id, int64_t delta, int mode) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->set_pos_rel(trans->id, (int32_t)delta, mode); +} + +static int trans_push_back_byte(void *id, int c) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->push_back_byte(trans->id, c); +} + +static int64_t trans_get_length(void *id) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->get_length(trans->id); +} + +static int trans_can_seek(void *id) +{ + WavpackReaderTranslator *trans = id; + return trans->reader->can_seek(trans->id); +} + +static int trans_close_stream(void *id) +{ + free(id); + return 0; +} + +static WavpackStreamReader64 trans_reader = { + trans_read_bytes, trans_write_bytes, trans_get_pos, trans_set_pos_abs, trans_set_pos_rel, trans_push_back_byte, + trans_get_length, trans_can_seek, NULL, trans_close_stream}; + +// This function is identical to WavpackOpenFileInput64() except that instead +// of providing the new 64-bit reader callbacks, the old reader callbacks are +// utilized and a translation layer is employed. It is provided as a compatibility +// function for existing applications. To ensure that streaming applications using +// this function continue to work, the OPEN_NO_CHECKSUM flag is forced on when +// the OPEN_STREAMING flag is set. + +WavpackContext *WavpackOpenFileInputEx(WavpackStreamReader *reader, void *wv_id, void *wvc_id, char *error, int flags, + int norm_offset) +{ + WavpackReaderTranslator *trans_wv = NULL, *trans_wvc = NULL; + + // this prevents existing streaming applications from failing if they try to pass + // in blocks that have been modified from the original (e.g., Matroska blocks) + + if(flags & OPEN_STREAMING) flags |= OPEN_NO_CHECKSUM; + + if(wv_id) + { + trans_wv = malloc(sizeof(WavpackReaderTranslator)); + trans_wv->reader = reader; + trans_wv->id = wv_id; + } + + if(wvc_id) + { + trans_wvc = malloc(sizeof(WavpackReaderTranslator)); + trans_wvc->reader = reader; + trans_wvc->id = wvc_id; + } + + return WavpackOpenFileInputEx64(&trans_reader, trans_wv, trans_wvc, error, flags, norm_offset); +} diff --git a/wavpack/open_utils.c b/wavpack/open_utils.c new file mode 100644 index 0000000..70cf387 --- /dev/null +++ b/wavpack/open_utils.c @@ -0,0 +1,1327 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2016 David Bryant. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// open_utils.c + +// This module provides all the code required to open an existing WavPack file +// for reading by using a reader callback mechanism (NOT a filename). This +// includes the code required to find and parse WavPack blocks, process any +// included metadata, and queue up the bitstreams containing the encoded audio +// data. It does not the actual code to unpack audio data and this was done so +// that programs that just want to query WavPack files for information (like, +// for example, taggers) don't need to link in a lot of unnecessary code. + +#include +#include + +#include "wavpack_local.h" + +// This function is identical to WavpackOpenFileInput() except that instead +// of providing a filename to open, the caller provides a pointer to a set of +// reader callbacks and instances of up to two streams. The first of these +// streams is required and contains the regular WavPack data stream; the second +// contains the "correction" file if desired. Unlike the standard open +// function which handles the correction file transparently, in this case it +// is the responsibility of the caller to be aware of correction files. + +static int seek_eof_information(WavpackContext *wpc, int64_t *final_index, int get_wrapper); + +WavpackContext *WavpackOpenFileInputEx64(WavpackStreamReader64 *reader, void *wv_id, void *wvc_id, char *error, + int flags, int norm_offset) +{ + WavpackContext *wpc = malloc(sizeof(WavpackContext)); + WavpackStream *wps; + int num_blocks = 0; + unsigned char first_byte; + uint32_t bcount; + + if(!wpc) + { + if(error) strcpy(error, "can't allocate memory"); + return NULL; + } + + CLEAR(*wpc); + wpc->wv_in = wv_id; + wpc->wvc_in = wvc_id; + wpc->reader = reader; + wpc->total_samples = -1; + wpc->norm_offset = norm_offset; + wpc->max_streams = OLD_MAX_STREAMS; // use this until overwritten with actual number + wpc->open_flags = flags; + + wpc->filelen = wpc->reader->get_length(wpc->wv_in); + +#ifndef NO_TAGS + if((flags & (OPEN_TAGS | OPEN_EDIT_TAGS)) && wpc->reader->can_seek(wpc->wv_in)) + { + load_tag(wpc); + wpc->reader->set_pos_abs(wpc->wv_in, 0); + + if((flags & OPEN_EDIT_TAGS) && !editable_tag(&wpc->m_tag)) + { + if(error) strcpy(error, "can't edit tags located at the beginning of files!"); + return WavpackCloseFile(wpc); + } + } +#endif + + if(wpc->reader->read_bytes(wpc->wv_in, &first_byte, 1) != 1) + { + if(error) strcpy(error, "can't read all of WavPack file!"); + return WavpackCloseFile(wpc); + } + + wpc->reader->push_back_byte(wpc->wv_in, first_byte); + + if(first_byte == 'R') + { +#ifdef ENABLE_LEGACY + return open_file3(wpc, error); +#else + if(error) strcpy(error, "this legacy WavPack file is deprecated, use version 4.80.0 to transcode"); + return WavpackCloseFile(wpc); +#endif + } + + wpc->streams = malloc((wpc->num_streams = 1) * sizeof(wpc->streams[0])); + if(!wpc->streams) + { + if(error) strcpy(error, "can't allocate memory"); + return WavpackCloseFile(wpc); + } + + wpc->streams[0] = wps = malloc(sizeof(WavpackStream)); + if(!wps) + { + if(error) strcpy(error, "can't allocate memory"); + return WavpackCloseFile(wpc); + } + CLEAR(*wps); + + while(!wps->wphdr.block_samples) + { + + wpc->filepos = wpc->reader->get_pos(wpc->wv_in); + bcount = read_next_header(wpc->reader, wpc->wv_in, &wps->wphdr); + + if(bcount == (uint32_t)-1 || (!wps->wphdr.block_samples && num_blocks++ > 16)) + { + if(error) strcpy(error, "not compatible with this version of WavPack file!"); + return WavpackCloseFile(wpc); + } + + wpc->filepos += bcount; + wps->blockbuff = malloc(wps->wphdr.ckSize + 8); + if(!wps->blockbuff) + { + if(error) strcpy(error, "can't allocate memory"); + return WavpackCloseFile(wpc); + } + memcpy(wps->blockbuff, &wps->wphdr, 32); + + if(wpc->reader->read_bytes(wpc->wv_in, wps->blockbuff + 32, wps->wphdr.ckSize - 24) != wps->wphdr.ckSize - 24) + { + if(error) strcpy(error, "can't read all of WavPack file!"); + return WavpackCloseFile(wpc); + } + + // if block does not verify, flag error, free buffer, and continue + if(!WavpackVerifySingleBlock(wps->blockbuff, !(flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.block_samples = 0; + free(wps->blockbuff); + wps->blockbuff = NULL; + wpc->crc_errors++; + continue; + } + + wps->init_done = FALSE; + + if(wps->wphdr.block_samples) + { + if(flags & OPEN_STREAMING) + SET_BLOCK_INDEX(wps->wphdr, 0); + else if(wpc->total_samples == -1) + { + if(GET_BLOCK_INDEX(wps->wphdr) || GET_TOTAL_SAMPLES(wps->wphdr) == -1) + { + wpc->initial_index = GET_BLOCK_INDEX(wps->wphdr); + SET_BLOCK_INDEX(wps->wphdr, 0); + + if(wpc->reader->can_seek(wpc->wv_in)) + { + int64_t final_index = -1; + + seek_eof_information(wpc, &final_index, FALSE); + + if(final_index != -1) wpc->total_samples = final_index - wpc->initial_index; + } + } + else + wpc->total_samples = GET_TOTAL_SAMPLES(wps->wphdr); + } + } + else if(wpc->total_samples == -1 && !GET_BLOCK_INDEX(wps->wphdr) && GET_TOTAL_SAMPLES(wps->wphdr)) + wpc->total_samples = GET_TOTAL_SAMPLES(wps->wphdr); + + if(wpc->wvc_in && wps->wphdr.block_samples && (wps->wphdr.flags & HYBRID_FLAG)) + { + unsigned char ch; + + if(wpc->reader->read_bytes(wpc->wvc_in, &ch, 1) == 1) + { + wpc->reader->push_back_byte(wpc->wvc_in, ch); + wpc->file2len = wpc->reader->get_length(wpc->wvc_in); + wpc->wvc_flag = TRUE; + } + } + + if(wpc->wvc_flag && !read_wvc_block(wpc)) + { + if(error) strcpy(error, "not compatible with this version of correction file!"); + return WavpackCloseFile(wpc); + } + + if(!wps->init_done && !unpack_init(wpc)) + { + if(error) + strcpy(error, wpc->error_message[0] ? wpc->error_message + : "not compatible with this version of WavPack file!"); + + return WavpackCloseFile(wpc); + } + + wps->init_done = TRUE; + } + + wpc->config.flags &= ~0xff; + wpc->config.flags |= wps->wphdr.flags & 0xff; + + if(!wpc->config.num_channels) + { + wpc->config.num_channels = (wps->wphdr.flags & MONO_FLAG) ? 1 : 2; + wpc->config.channel_mask = 0x5 - wpc->config.num_channels; + } + + if((flags & OPEN_2CH_MAX) && !(wps->wphdr.flags & FINAL_BLOCK)) + wpc->reduced_channels = (wps->wphdr.flags & MONO_FLAG) ? 1 : 2; + + if(wps->wphdr.flags & DSD_FLAG) + { +#ifdef ENABLE_DSD + if(flags & OPEN_DSD_NATIVE) + { + wpc->config.bytes_per_sample = 1; + wpc->config.bits_per_sample = 8; + } + else if(flags & OPEN_DSD_AS_PCM) + { + wpc->decimation_context = + decimate_dsd_init(wpc->reduced_channels ? wpc->reduced_channels : wpc->config.num_channels); + + wpc->config.bytes_per_sample = 3; + wpc->config.bits_per_sample = 24; + } + else + { + if(error) strcpy(error, "not configured to handle DSD WavPack files!"); + return WavpackCloseFile(wpc); + } +#else + if(error) strcpy(error, "not configured to handle DSD WavPack files!"); + return WavpackCloseFile(wpc); +#endif + } + else + { + wpc->config.bytes_per_sample = (wps->wphdr.flags & BYTES_STORED) + 1; + wpc->config.float_norm_exp = wps->float_norm_exp; + + wpc->config.bits_per_sample = + (wpc->config.bytes_per_sample * 8) - ((wps->wphdr.flags & SHIFT_MASK) >> SHIFT_LSB); + } + + if(!wpc->config.sample_rate) + { + if(!wps->wphdr.block_samples || (wps->wphdr.flags & SRATE_MASK) == SRATE_MASK) + wpc->config.sample_rate = 44100; + else + wpc->config.sample_rate = sample_rates[(wps->wphdr.flags & SRATE_MASK) >> SRATE_LSB]; + } + + return wpc; +} + +// This function returns the major version number of the WavPack program +// (or library) that created the open file. Currently, this can be 1 to 5. +// Minor versions are not recorded in WavPack files. + +int WavpackGetVersion(WavpackContext *wpc) +{ + if(wpc) + { +#ifdef ENABLE_LEGACY + if(wpc->stream3) return get_version3(wpc); +#endif + return wpc->version_five ? 5 : 4; + } + + return 0; +} + +// Return the file format specified in the call to WavpackSetFileInformation() +// when the file was created. For all files created prior to WavPack 5.0 this +// will 0 (WP_FORMAT_WAV). + +unsigned char WavpackGetFileFormat(WavpackContext *wpc) { return wpc->file_format; } + +// Return a string representing the recommended file extension for the open +// WavPack file. For all files created prior to WavPack 5.0 this will be "wav", +// even for raw files with no RIFF into. This string is specified in the +// call to WavpackSetFileInformation() when the file was created. + +char *WavpackGetFileExtension(WavpackContext *wpc) +{ + if(wpc && wpc->file_extension[0]) + return wpc->file_extension; + else + return "wav"; +} + +// This function initializes everything required to unpack a WavPack block +// and must be called before unpack_samples() is called to obtain audio data. +// It is assumed that the WavpackHeader has been read into the wps->wphdr +// (in the current WavpackStream) and that the entire block has been read at +// wps->blockbuff. If a correction file is available (wpc->wvc_flag = TRUE) +// then the corresponding correction block must be read into wps->block2buff +// and its WavpackHeader has overwritten the header at wps->wphdr. This is +// where all the metadata blocks are scanned including those that contain +// bitstream data. + +static int read_metadata_buff(WavpackMetadata *wpmd, unsigned char *blockbuff, unsigned char **buffptr); +static int process_metadata(WavpackContext *wpc, WavpackMetadata *wpmd); +static void bs_open_read(Bitstream *bs, void *buffer_start, void *buffer_end); + +int unpack_init(WavpackContext *wpc) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + unsigned char *blockptr, *block2ptr; + WavpackMetadata wpmd; + + wps->num_terms = 0; + wps->mute_error = FALSE; + wps->crc = wps->crc_x = 0xffffffff; + wps->dsd.ready = 0; + CLEAR(wps->wvbits); + CLEAR(wps->wvcbits); + CLEAR(wps->wvxbits); + CLEAR(wps->decorr_passes); + CLEAR(wps->dc); + CLEAR(wps->w); + + if(!(wps->wphdr.flags & MONO_FLAG) && wpc->config.num_channels && wps->wphdr.block_samples && + (wpc->reduced_channels == 1 || wpc->config.num_channels == 1)) + { + wps->mute_error = TRUE; + return FALSE; + } + + if((wps->wphdr.flags & UNKNOWN_FLAGS) || (wps->wphdr.flags & MONO_DATA) == MONO_DATA) + { + wps->mute_error = TRUE; + return FALSE; + } + + blockptr = wps->blockbuff + sizeof(WavpackHeader); + + while(read_metadata_buff(&wpmd, wps->blockbuff, &blockptr)) + if(!process_metadata(wpc, &wpmd)) + { + wps->mute_error = TRUE; + return FALSE; + } + + if(wps->wphdr.block_samples && wpc->wvc_flag && wps->block2buff) + { + block2ptr = wps->block2buff + sizeof(WavpackHeader); + + while(read_metadata_buff(&wpmd, wps->block2buff, &block2ptr)) + if(!process_metadata(wpc, &wpmd)) + { + wps->mute_error = TRUE; + return FALSE; + } + } + + if(wps->wphdr.block_samples && ((wps->wphdr.flags & DSD_FLAG) ? !wps->dsd.ready : !bs_is_open(&wps->wvbits))) + { + if(bs_is_open(&wps->wvcbits)) strcpy(wpc->error_message, "can't unpack correction files alone!"); + + wps->mute_error = TRUE; + return FALSE; + } + + if(wps->wphdr.block_samples && !bs_is_open(&wps->wvxbits)) + { + if((wps->wphdr.flags & INT32_DATA) && wps->int32_sent_bits) wpc->lossy_blocks = TRUE; + + if((wps->wphdr.flags & FLOAT_DATA) && + wps->float_flags & (FLOAT_EXCEPTIONS | FLOAT_ZEROS_SENT | FLOAT_SHIFT_SENT | FLOAT_SHIFT_SAME)) + wpc->lossy_blocks = TRUE; + } + + if(wps->wphdr.block_samples) wps->sample_index = GET_BLOCK_INDEX(wps->wphdr); + + return TRUE; +} + +//////////////////////////////// matadata handlers /////////////////////////////// + +// These functions handle specific metadata types and are called directly +// during WavPack block parsing by process_metadata() at the bottom. + +// This function initialzes the main bitstream for audio samples, which must +// be in the "wv" file. + +static int init_wv_bitstream(WavpackStream *wps, WavpackMetadata *wpmd) +{ + if(!wpmd->byte_length || (wpmd->byte_length & 1)) return FALSE; + + bs_open_read(&wps->wvbits, wpmd->data, (unsigned char *)wpmd->data + wpmd->byte_length); + return TRUE; +} + +// This function initialzes the "correction" bitstream for audio samples, +// which currently must be in the "wvc" file. + +static int init_wvc_bitstream(WavpackStream *wps, WavpackMetadata *wpmd) +{ + if(!wpmd->byte_length || (wpmd->byte_length & 1)) return FALSE; + + bs_open_read(&wps->wvcbits, wpmd->data, (unsigned char *)wpmd->data + wpmd->byte_length); + return TRUE; +} + +// This function initialzes the "extra" bitstream for audio samples which +// contains the information required to losslessly decompress 32-bit float data +// or integer data that exceeds 24 bits. This bitstream is in the "wv" file +// for pure lossless data or the "wvc" file for hybrid lossless. This data +// would not be used for hybrid lossy mode. There is also a 32-bit CRC stored +// in the first 4 bytes of these blocks. + +static int init_wvx_bitstream(WavpackStream *wps, WavpackMetadata *wpmd) +{ + unsigned char *cp = wpmd->data; + + if(wpmd->byte_length <= 4 || (wpmd->byte_length & 1)) return FALSE; + + wps->crc_wvx = *cp++; + wps->crc_wvx |= (int32_t)*cp++ << 8; + wps->crc_wvx |= (int32_t)*cp++ << 16; + wps->crc_wvx |= (int32_t)*cp++ << 24; + + bs_open_read(&wps->wvxbits, cp, (unsigned char *)wpmd->data + wpmd->byte_length); + return TRUE; +} + +// Read the int32 data from the specified metadata into the specified stream. +// This data is used for integer data that has more than 24 bits of magnitude +// or, in some cases, used to eliminate redundant bits from any audio stream. + +static int read_int32_info(WavpackStream *wps, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length; + char *byteptr = wpmd->data; + + if(bytecnt != 4) return FALSE; + + wps->int32_sent_bits = *byteptr++; + wps->int32_zeros = *byteptr++; + wps->int32_ones = *byteptr++; + wps->int32_dups = *byteptr; + + return TRUE; +} + +static int read_float_info(WavpackStream *wps, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length; + char *byteptr = wpmd->data; + + if(bytecnt != 4) return FALSE; + + wps->float_flags = *byteptr++; + wps->float_shift = *byteptr++; + wps->float_max_exp = *byteptr++; + wps->float_norm_exp = *byteptr; + return TRUE; +} + +// Read multichannel information from metadata. The first byte is the total +// number of channels and the following bytes represent the channel_mask +// as described for Microsoft WAVEFORMATEX. + +static int read_channel_info(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length, shift = 0, mask_bits; + unsigned char *byteptr = wpmd->data; + uint32_t mask = 0; + + if(!bytecnt || bytecnt > 7) return FALSE; + + if(!wpc->config.num_channels) + { + + // if bytecnt is 6 or 7 we are using new configuration with "unlimited" streams + + if(bytecnt >= 6) + { + wpc->config.num_channels = (byteptr[0] | ((byteptr[2] & 0xf) << 8)) + 1; + wpc->max_streams = (byteptr[1] | ((byteptr[2] & 0xf0) << 4)) + 1; + + if(wpc->config.num_channels < wpc->max_streams) return FALSE; + + byteptr += 3; + mask = *byteptr++; + mask |= (uint32_t)*byteptr++ << 8; + mask |= (uint32_t)*byteptr++ << 16; + + if(bytecnt == 7) // this was introduced in 5.0 + mask |= (uint32_t)*byteptr << 24; + } + else + { + wpc->config.num_channels = *byteptr++; + + while(--bytecnt) + { + mask |= (uint32_t)*byteptr++ << shift; + shift += 8; + } + } + + if(wpc->config.num_channels > wpc->max_streams * 2) return FALSE; + + wpc->config.channel_mask = mask; + + for(mask_bits = 0; mask; mask >>= 1) + if((mask & 1) && ++mask_bits > wpc->config.num_channels) return FALSE; + } + + return TRUE; +} + +// Read multichannel identity information from metadata. Data is an array of +// unsigned characters representing any channels in the file that DO NOT +// match one the 18 Microsoft standard channels (and are represented in the +// channel mask). A value of 0 is not allowed and 0xff means an unknown or +// undefined channel identity. + +static int read_channel_identities(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + if(!wpc->channel_identities) + { + wpc->channel_identities = malloc(wpmd->byte_length + 1); + memcpy(wpc->channel_identities, wpmd->data, wpmd->byte_length); + wpc->channel_identities[wpmd->byte_length] = 0; + } + + return TRUE; +} + +// Read configuration information from metadata. + +static int read_config_info(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length; + unsigned char *byteptr = wpmd->data; + + if(bytecnt >= 3) + { + wpc->config.flags &= 0xff; + wpc->config.flags |= (int32_t)*byteptr++ << 8; + wpc->config.flags |= (int32_t)*byteptr++ << 16; + wpc->config.flags |= (int32_t)*byteptr++ << 24; + bytecnt -= 3; + + if(bytecnt && (wpc->config.flags & CONFIG_EXTRA_MODE)) + { + wpc->config.xmode = *byteptr++; + bytecnt--; + } + + // we used an extra config byte here for the 5.0.0 alpha, so still + // honor it now (but this has been replaced with NEW_CONFIG) + + if(bytecnt) + { + wpc->config.qmode = (wpc->config.qmode & ~0xff) | *byteptr; + wpc->version_five = 1; + } + } + + return TRUE; +} + +// Read "new" configuration information from metadata. + +static int read_new_config_info(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length; + unsigned char *byteptr = wpmd->data; + + wpc->version_five = 1; // just having this block signals version 5.0 + + wpc->file_format = wpc->config.qmode = wpc->channel_layout = 0; + + if(wpc->channel_reordering) + { + free(wpc->channel_reordering); + wpc->channel_reordering = NULL; + } + + // if there's any data, the first two bytes are file_format and qmode flags + + if(bytecnt >= 2) + { + wpc->file_format = *byteptr++; + wpc->config.qmode = (wpc->config.qmode & ~0xff) | *byteptr++; + bytecnt -= 2; + + // another byte indicates a channel layout + + if(bytecnt) + { + int nchans, i; + + wpc->channel_layout = (int32_t)*byteptr++ << 16; + bytecnt--; + + // another byte means we have a channel count for the layout and maybe a reordering + + if(bytecnt) + { + wpc->channel_layout += nchans = *byteptr++; + bytecnt--; + + // any more means there's a reordering string + + if(bytecnt) + { + if(bytecnt > nchans) return FALSE; + + wpc->channel_reordering = malloc(nchans); + + // note that redundant reordering info is not stored, so we fill in the rest + + if(wpc->channel_reordering) + { + for(i = 0; i < nchans; ++i) + if(bytecnt) + { + wpc->channel_reordering[i] = *byteptr++; + + if(wpc->channel_reordering[i] >= nchans) // make sure index is in range + wpc->channel_reordering[i] = 0; + + bytecnt--; + } + else + wpc->channel_reordering[i] = i; + } + } + } + else + wpc->channel_layout += wpc->config.num_channels; + } + } + + return TRUE; +} + +// Read non-standard sampling rate from metadata. + +static int read_sample_rate(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + int bytecnt = wpmd->byte_length; + unsigned char *byteptr = wpmd->data; + + if(bytecnt == 3 || bytecnt == 4) + { + wpc->config.sample_rate = (int32_t)*byteptr++; + wpc->config.sample_rate |= (int32_t)*byteptr++ << 8; + wpc->config.sample_rate |= (int32_t)*byteptr++ << 16; + + // for sampling rates > 16777215 (non-audio probably, or ...) + + if(bytecnt == 4) wpc->config.sample_rate |= (int32_t)(*byteptr & 0x7f) << 24; + } + + return TRUE; +} + +// Read wrapper data from metadata. Currently, this consists of the RIFF +// header and trailer that wav files contain around the audio data but could +// be used for other formats as well. Because WavPack files contain all the +// information required for decoding and playback, this data can probably +// be ignored except when an exact wavefile restoration is needed. + +static int read_wrapper_data(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + if((wpc->open_flags & OPEN_WRAPPER) && wpc->wrapper_bytes < MAX_WRAPPER_BYTES && wpmd->byte_length) + { + wpc->wrapper_data = realloc(wpc->wrapper_data, wpc->wrapper_bytes + wpmd->byte_length); + if(!wpc->wrapper_data) return FALSE; + memcpy(wpc->wrapper_data + wpc->wrapper_bytes, wpmd->data, wpmd->byte_length); + wpc->wrapper_bytes += wpmd->byte_length; + } + + return TRUE; +} + +static int read_metadata_buff(WavpackMetadata *wpmd, unsigned char *blockbuff, unsigned char **buffptr) +{ + WavpackHeader *wphdr = (WavpackHeader *)blockbuff; + unsigned char *buffend = blockbuff + wphdr->ckSize + 8; + + if(buffend - *buffptr < 2) return FALSE; + + wpmd->id = *(*buffptr)++; + wpmd->byte_length = *(*buffptr)++ << 1; + + if(wpmd->id & ID_LARGE) + { + wpmd->id &= ~ID_LARGE; + + if(buffend - *buffptr < 2) return FALSE; + + wpmd->byte_length += *(*buffptr)++ << 9; + wpmd->byte_length += *(*buffptr)++ << 17; + } + + if(wpmd->id & ID_ODD_SIZE) + { + if(!wpmd->byte_length) // odd size and zero length makes no sense + return FALSE; + wpmd->id &= ~ID_ODD_SIZE; + wpmd->byte_length--; + } + + if(wpmd->byte_length) + { + if(buffend - *buffptr < wpmd->byte_length + (wpmd->byte_length & 1)) + { + wpmd->data = NULL; + return FALSE; + } + + wpmd->data = *buffptr; + (*buffptr) += wpmd->byte_length + (wpmd->byte_length & 1); + } + else + wpmd->data = NULL; + + return TRUE; +} + +static int process_metadata(WavpackContext *wpc, WavpackMetadata *wpmd) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + + switch(wpmd->id) + { + case ID_DUMMY: + return TRUE; + + case ID_DECORR_TERMS: + return read_decorr_terms(wps, wpmd); + + case ID_DECORR_WEIGHTS: + return read_decorr_weights(wps, wpmd); + + case ID_DECORR_SAMPLES: + return read_decorr_samples(wps, wpmd); + + case ID_ENTROPY_VARS: + return read_entropy_vars(wps, wpmd); + + case ID_HYBRID_PROFILE: + return read_hybrid_profile(wps, wpmd); + + case ID_SHAPING_WEIGHTS: + return read_shaping_info(wps, wpmd); + + case ID_FLOAT_INFO: + return read_float_info(wps, wpmd); + + case ID_INT32_INFO: + return read_int32_info(wps, wpmd); + + case ID_CHANNEL_INFO: + return read_channel_info(wpc, wpmd); + + case ID_CHANNEL_IDENTITIES: + return read_channel_identities(wpc, wpmd); + + case ID_CONFIG_BLOCK: + return read_config_info(wpc, wpmd); + + case ID_NEW_CONFIG_BLOCK: + return read_new_config_info(wpc, wpmd); + + case ID_SAMPLE_RATE: + return read_sample_rate(wpc, wpmd); + + case ID_WV_BITSTREAM: + return init_wv_bitstream(wps, wpmd); + + case ID_WVC_BITSTREAM: + return init_wvc_bitstream(wps, wpmd); + + case ID_WVX_BITSTREAM: + return init_wvx_bitstream(wps, wpmd); + + case ID_DSD_BLOCK: +#ifdef ENABLE_DSD + return init_dsd_block(wpc, wpmd); +#else + strcpy(wpc->error_message, "not configured to handle DSD WavPack files!"); + return FALSE; +#endif + + case ID_ALT_HEADER: + case ID_ALT_TRAILER: + if(!(wpc->open_flags & OPEN_ALT_TYPES)) return TRUE; + + case ID_RIFF_HEADER: + case ID_RIFF_TRAILER: + return read_wrapper_data(wpc, wpmd); + + case ID_ALT_MD5_CHECKSUM: + if(!(wpc->open_flags & OPEN_ALT_TYPES)) return TRUE; + + case ID_MD5_CHECKSUM: + if(wpmd->byte_length == 16) + { + memcpy(wpc->config.md5_checksum, wpmd->data, 16); + wpc->config.flags |= CONFIG_MD5_CHECKSUM; + wpc->config.md5_read = 1; + } + + return TRUE; + + case ID_ALT_EXTENSION: + if(wpmd->byte_length && wpmd->byte_length < sizeof(wpc->file_extension)) + { + memcpy(wpc->file_extension, wpmd->data, wpmd->byte_length); + wpc->file_extension[wpmd->byte_length] = 0; + } + + return TRUE; + + // we don't actually verify the checksum here (it's done right after the + // block is read), but it's a good indicator of version 5 files + + case ID_BLOCK_CHECKSUM: + wpc->version_five = 1; + return TRUE; + + default: + return (wpmd->id & ID_OPTIONAL_DATA) ? TRUE : FALSE; + } +} + +//////////////////////////////// bitstream management /////////////////////////////// + +// Open the specified BitStream and associate with the specified buffer. + +static void bs_read(Bitstream *bs); + +static void bs_open_read(Bitstream *bs, void *buffer_start, void *buffer_end) +{ + bs->error = bs->sr = bs->bc = 0; + bs->ptr = (bs->buf = buffer_start) - 1; + bs->end = buffer_end; + bs->wrap = bs_read; +} + +// This function is only called from the getbit() and getbits() macros when +// the BitStream has been exhausted and more data is required. Sinve these +// bistreams no longer access files, this function simple sets an error and +// resets the buffer. + +static void bs_read(Bitstream *bs) +{ + bs->ptr = bs->buf; + bs->error = 1; +} + +// This function is called to close the bitstream. It returns the number of +// full bytes actually read as bits. + +uint32_t bs_close_read(Bitstream *bs) +{ + uint32_t bytes_read; + + if(bs->bc < sizeof(*(bs->ptr)) * 8) bs->ptr++; + + bytes_read = (uint32_t)(bs->ptr - bs->buf) * sizeof(*(bs->ptr)); + + if(!(bytes_read & 1)) ++bytes_read; + + CLEAR(*bs); + return bytes_read; +} + +// Normally the trailing wrapper will not be available when a WavPack file is first +// opened for reading because it is stored in the final block of the file. This +// function forces a seek to the end of the file to pick up any trailing wrapper +// stored there (then use WavPackGetWrapper**() to obtain). This can obviously only +// be used for seekable files (not pipes) and is not available for pre-4.0 WavPack +// files. + +void WavpackSeekTrailingWrapper(WavpackContext *wpc) +{ + if((wpc->open_flags & OPEN_WRAPPER) && wpc->reader->can_seek(wpc->wv_in) && !wpc->stream3) + seek_eof_information(wpc, NULL, TRUE); +} + +// Get any MD5 checksum stored in the metadata (should be called after reading +// last sample or an extra seek will occur). A return value of FALSE indicates +// that no MD5 checksum was stored. + +int WavpackGetMD5Sum(WavpackContext *wpc, unsigned char data[16]) +{ + if(wpc->config.flags & CONFIG_MD5_CHECKSUM) + { + if(!wpc->config.md5_read && wpc->reader->can_seek(wpc->wv_in)) seek_eof_information(wpc, NULL, FALSE); + + if(wpc->config.md5_read) + { + memcpy(data, wpc->config.md5_checksum, 16); + return TRUE; + } + } + + return FALSE; +} + +// Read from current file position until a valid 32-byte WavPack 4.0 header is +// found and read into the specified pointer. The number of bytes skipped is +// returned. If no WavPack header is found within 1 meg, then a -1 is returned +// to indicate the error. No additional bytes are read past the header and it +// is returned in the processor's native endian mode. Seeking is not required. + +uint32_t read_next_header(WavpackStreamReader64 *reader, void *id, WavpackHeader *wphdr) +{ + unsigned char buffer[sizeof(*wphdr)], *sp = buffer + sizeof(*wphdr), *ep = sp; + uint32_t bytes_skipped = 0; + int bleft; + + while(1) + { + if(sp < ep) + { + bleft = (int)(ep - sp); + memmove(buffer, sp, bleft); + } + else + bleft = 0; + + if(reader->read_bytes(id, buffer + bleft, sizeof(*wphdr) - bleft) != sizeof(*wphdr) - bleft) return -1; + + sp = buffer; + + if(*sp++ == 'w' && *sp == 'v' && *++sp == 'p' && *++sp == 'k' && !(*++sp & 1) && sp[2] < 16 && !sp[3] && + (sp[2] || sp[1] || *sp >= 24) && sp[5] == 4 && sp[4] >= (MIN_STREAM_VERS & 0xff) && + sp[4] <= (MAX_STREAM_VERS & 0xff) && sp[18] < 3 && !sp[19]) + { + memcpy(wphdr, buffer, sizeof(*wphdr)); + WavpackLittleEndianToNative(wphdr, WavpackHeaderFormat); + return bytes_skipped; + } + + while(sp < ep && *sp != 'w') sp++; + + if((bytes_skipped += (uint32_t)(sp - buffer)) > 1024 * 1024) return -1; + } +} + +// Compare the regular wv file block header to a potential matching wvc +// file block header and return action code based on analysis: +// +// 0 = use wvc block (assuming rest of block is readable) +// 1 = bad match; try to read next wvc block +// -1 = bad match; ignore wvc file for this block and backup fp (if +// possible) and try to use this block next time + +static int match_wvc_header(WavpackHeader *wv_hdr, WavpackHeader *wvc_hdr) +{ + if(GET_BLOCK_INDEX(*wv_hdr) == GET_BLOCK_INDEX(*wvc_hdr) && wv_hdr->block_samples == wvc_hdr->block_samples) + { + int wvi = 0, wvci = 0; + + if(wv_hdr->flags == wvc_hdr->flags) return 0; + + if(wv_hdr->flags & INITIAL_BLOCK) wvi -= 1; + + if(wv_hdr->flags & FINAL_BLOCK) wvi += 1; + + if(wvc_hdr->flags & INITIAL_BLOCK) wvci -= 1; + + if(wvc_hdr->flags & FINAL_BLOCK) wvci += 1; + + return (wvci - wvi < 0) ? 1 : -1; + } + + if(((GET_BLOCK_INDEX(*wvc_hdr) - GET_BLOCK_INDEX(*wv_hdr)) << 24) < 0) + return 1; + else + return -1; +} + +// Read the wvc block that matches the regular wv block that has been +// read for the current stream. If an exact match is not found then +// we either keep reading or back up and (possibly) use the block +// later. The skip_wvc flag is set if not matching wvc block is found +// so that we can still decode using only the lossy version (although +// we flag this as an error). A return of FALSE indicates a serious +// error (not just that we missed one wvc block). + +int read_wvc_block(WavpackContext *wpc) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + int64_t bcount, file2pos; + WavpackHeader orig_wphdr; + WavpackHeader wphdr; + int compare_result; + + while(1) + { + file2pos = wpc->reader->get_pos(wpc->wvc_in); + bcount = read_next_header(wpc->reader, wpc->wvc_in, &wphdr); + + if(bcount == (uint32_t)-1) + { + wps->wvc_skip = TRUE; + wpc->crc_errors++; + return FALSE; + } + + memcpy(&orig_wphdr, &wphdr, 32); // save original header for verify step + + if(wpc->open_flags & OPEN_STREAMING) + SET_BLOCK_INDEX(wphdr, wps->sample_index = 0); + else + SET_BLOCK_INDEX(wphdr, GET_BLOCK_INDEX(wphdr) - wpc->initial_index); + + if(wphdr.flags & INITIAL_BLOCK) wpc->file2pos = file2pos + bcount; + + compare_result = match_wvc_header(&wps->wphdr, &wphdr); + + if(!compare_result) + { + wps->block2buff = malloc(wphdr.ckSize + 8); + if(!wps->block2buff) return FALSE; + + if(wpc->reader->read_bytes(wpc->wvc_in, wps->block2buff + 32, wphdr.ckSize - 24) != wphdr.ckSize - 24) + { + free(wps->block2buff); + wps->block2buff = NULL; + wps->wvc_skip = TRUE; + wpc->crc_errors++; + return FALSE; + } + + memcpy(wps->block2buff, &orig_wphdr, 32); + + // don't use corrupt blocks + if(!WavpackVerifySingleBlock(wps->block2buff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + free(wps->block2buff); + wps->block2buff = NULL; + wps->wvc_skip = TRUE; + wpc->crc_errors++; + return TRUE; + } + + wps->wvc_skip = FALSE; + memcpy(wps->block2buff, &wphdr, 32); + memcpy(&wps->wphdr, &wphdr, 32); + return TRUE; + } + else if(compare_result == -1) + { + wps->wvc_skip = TRUE; + wpc->reader->set_pos_rel(wpc->wvc_in, -32, SEEK_CUR); + wpc->crc_errors++; + return TRUE; + } + } +} + +// This function is used to seek to end of a file to obtain certain information +// that is stored there at the file creation time because it is not known at +// the start. This includes the MD5 sum and and trailing part of the file +// wrapper, and in some rare cases may include the total number of samples in +// the file (although we usually try to back up and write that at the front of +// the file). Note this function restores the file position to its original +// location (and obviously requires a seekable file). The normal return value +// is TRUE indicating no errors, although this does not actually mean that any +// information was retrieved. An error return of FALSE usually means the file +// terminated unexpectedly. Note that this could be used to get all three +// types of information in one go, but it's not actually used that way now. + +static int seek_eof_information(WavpackContext *wpc, int64_t *final_index, int get_wrapper) +{ + int64_t restore_pos, last_pos = -1; + WavpackStreamReader64 *reader = wpc->reader; + int alt_types = wpc->open_flags & OPEN_ALT_TYPES; + uint32_t blocks = 0, audio_blocks = 0; + void *id = wpc->wv_in; + WavpackHeader wphdr; + + restore_pos = reader->get_pos(id); // we restore file position when done + + // start 1MB from the end-of-file, or from the start if the file is not that big + + if(reader->get_length(id) > (int64_t)1048576) + reader->set_pos_rel(id, -1048576, SEEK_END); + else + reader->set_pos_abs(id, 0); + + // Note that we go backward (without parsing inside blocks) until we find a block + // with audio (careful to not get stuck in a loop). Only then do we go forward + // parsing all blocks in their entirety. + + while(1) + { + uint32_t bcount = read_next_header(reader, id, &wphdr); + int64_t current_pos = reader->get_pos(id); + + // if we just got to the same place as last time, we're stuck and need to give up + + if(current_pos == last_pos) + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + + last_pos = current_pos; + + // We enter here if we just read 1 MB without seeing any WavPack block headers. + // Since WavPack blocks are < 1 MB, that means we're in a big APE tag, or we got + // to the end-of-file. + + if(bcount == (uint32_t)-1) + { + + // if we have not seen any blocks at all yet, back up almost 2 MB (or to the + // beginning of the file) and try again + + if(!blocks) + { + if(current_pos > (int64_t)2000000) + reader->set_pos_rel(id, -2000000, SEEK_CUR); + else + reader->set_pos_abs(id, 0); + + continue; + } + + // if we have seen WavPack blocks, then this means we've done all we can do here + + reader->set_pos_abs(id, restore_pos); + return TRUE; + } + + blocks++; + + // If the block has audio samples, calculate a final index, although this is not + // final since this may not be the last block with audio. On the other hand, if + // this block does not have audio, and we haven't seen one with audio, we have + // to go back some more. + + if(wphdr.block_samples) + { + if(final_index) *final_index = GET_BLOCK_INDEX(wphdr) + wphdr.block_samples; + + audio_blocks++; + } + else if(!audio_blocks) + { + if(current_pos > (int64_t)1048576) + reader->set_pos_rel(id, -1048576, SEEK_CUR); + else + reader->set_pos_abs(id, 0); + + continue; + } + + // at this point we have seen at least one block with audio, so we parse the + // entire block looking for MD5 metadata or (conditionally) trailing wrappers + + bcount = wphdr.ckSize - sizeof(WavpackHeader) + 8; + + while(bcount >= 2) + { + unsigned char meta_id, c1, c2; + uint32_t meta_bc, meta_size; + + if(reader->read_bytes(id, &meta_id, 1) != 1 || reader->read_bytes(id, &c1, 1) != 1) + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + + meta_bc = c1 << 1; + bcount -= 2; + + if(meta_id & ID_LARGE) + { + if(bcount < 2 || reader->read_bytes(id, &c1, 1) != 1 || reader->read_bytes(id, &c2, 1) != 1) + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + + meta_bc += ((uint32_t)c1 << 9) + ((uint32_t)c2 << 17); + bcount -= 2; + } + + meta_size = (meta_id & ID_ODD_SIZE) ? meta_bc - 1 : meta_bc; + meta_id &= ID_UNIQUE; + + if(get_wrapper && (meta_id == ID_RIFF_TRAILER || (alt_types && meta_id == ID_ALT_TRAILER)) && meta_bc) + { + wpc->wrapper_data = realloc(wpc->wrapper_data, wpc->wrapper_bytes + meta_bc); + + if(!wpc->wrapper_data) + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + + if(reader->read_bytes(id, wpc->wrapper_data + wpc->wrapper_bytes, meta_bc) == meta_bc) + wpc->wrapper_bytes += meta_size; + else + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + } + else if(meta_id == ID_MD5_CHECKSUM || (alt_types && meta_id == ID_ALT_MD5_CHECKSUM)) + { + if(meta_bc == 16 && bcount >= 16) + { + if(reader->read_bytes(id, wpc->config.md5_checksum, 16) == 16) + wpc->config.md5_read = TRUE; + else + { + reader->set_pos_abs(id, restore_pos); + return FALSE; + } + } + else + reader->set_pos_rel(id, meta_bc, SEEK_CUR); + } + else + reader->set_pos_rel(id, meta_bc, SEEK_CUR); + + bcount -= meta_bc; + } + } +} + +// Quickly verify the referenced block. It is assumed that the WavPack header has been converted +// to native endian format. If a block checksum is performed, that is done in little-endian +// (file) format. It is also assumed that the caller has made sure that the block length +// indicated in the header is correct (we won't overflow the buffer). If a checksum is present, +// then it is checked, otherwise we just check that all the metadata blocks are formatted +// correctly (without looking at their contents). Returns FALSE for bad block. + +int WavpackVerifySingleBlock(unsigned char *buffer, int verify_checksum) +{ + WavpackHeader *wphdr = (WavpackHeader *)buffer; + uint32_t checksum_passed = 0, bcount, meta_bc; + unsigned char *dp, meta_id, c1, c2; + + if(strncmp(wphdr->ckID, "wvpk", 4) || wphdr->ckSize + 8 < sizeof(WavpackHeader)) return FALSE; + + bcount = wphdr->ckSize - sizeof(WavpackHeader) + 8; + dp = (unsigned char *)(wphdr + 1); + + while(bcount >= 2) + { + meta_id = *dp++; + c1 = *dp++; + + meta_bc = c1 << 1; + bcount -= 2; + + if(meta_id & ID_LARGE) + { + if(bcount < 2) return FALSE; + + c1 = *dp++; + c2 = *dp++; + meta_bc += ((uint32_t)c1 << 9) + ((uint32_t)c2 << 17); + bcount -= 2; + } + + if(bcount < meta_bc) return FALSE; + + if(verify_checksum && (meta_id & ID_UNIQUE) == ID_BLOCK_CHECKSUM) + { +#ifdef BITSTREAM_SHORTS + uint16_t *csptr = (uint16_t *)buffer; +#else + unsigned char *csptr = buffer; +#endif + int wcount = (int)(dp - 2 - buffer) >> 1; + uint32_t csum = (uint32_t)-1; + + if((meta_id & ID_ODD_SIZE) || meta_bc < 2 || meta_bc > 4) return FALSE; + +#ifdef BITSTREAM_SHORTS + while(wcount--) csum = (csum * 3) + *csptr++; +#else + WavpackNativeToLittleEndian((WavpackHeader *)buffer, WavpackHeaderFormat); + + while(wcount--) + { + csum = (csum * 3) + csptr[0] + (csptr[1] << 8); + csptr += 2; + } + + WavpackLittleEndianToNative((WavpackHeader *)buffer, WavpackHeaderFormat); +#endif + + if(meta_bc == 4) + { + if(*dp++ != (csum & 0xff) || *dp++ != ((csum >> 8) & 0xff) || *dp++ != ((csum >> 16) & 0xff) || + *dp++ != ((csum >> 24) & 0xff)) + return FALSE; + } + else + { + csum ^= csum >> 16; + + if(*dp++ != (csum & 0xff) || *dp++ != ((csum >> 8) & 0xff)) return FALSE; + } + + checksum_passed++; + } + + bcount -= meta_bc; + dp += meta_bc; + } + + return (bcount == 0) && (!verify_checksum || !(wphdr->flags & HAS_CHECKSUM) || checksum_passed); +} diff --git a/wavpack/read_words.c b/wavpack/read_words.c new file mode 100644 index 0000000..a4b1583 --- /dev/null +++ b/wavpack/read_words.c @@ -0,0 +1,637 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// read_words.c + +// This module provides entropy word decoding functions using +// a variation on the Rice method. This was introduced in version 3.93 +// because it allows splitting the data into a "lossy" stream and a +// "correction" stream in a very efficient manner and is therefore ideal +// for the "hybrid" mode. For 4.0, the efficiency of this method was +// significantly improved by moving away from the normal Rice restriction of +// using powers of two for the modulus divisions and now the method can be +// used for both hybrid and pure lossless encoding. + +// Samples are divided by median probabilities at 5/7 (71.43%), 10/49 (20.41%), +// and 20/343 (5.83%). Each zone has 3.5 times fewer samples than the +// previous. Using standard Rice coding on this data would result in 1.4 +// bits per sample average (not counting sign bit). However, there is a +// very simple encoding that is over 99% efficient with this data and +// results in about 1.22 bits per sample. + +#include +#include + +#include "wavpack_local.h" + +#if defined(HAVE___BUILTIN_CTZ) || defined(_WIN64) +#define USE_CTZ_OPTIMIZATION // use ctz intrinsic (or Windows equivalent) to count trailing ones +#else +#define USE_NEXT8_OPTIMIZATION // optimization using a table to count trailing ones +#endif + +#define USE_BITMASK_TABLES // use tables instead of shifting for certain masking operations + +///////////////////////////// local table storage //////////////////////////// + +#ifdef USE_NEXT8_OPTIMIZATION +static const char ones_count_table[] = { + 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, + 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, + 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, + 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, + 1, 0, 2, 0, 1, 0, 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, + 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 8}; +#endif + +///////////////////////////// executable code //////////////////////////////// + +static uint32_t __inline read_code(Bitstream *bs, uint32_t maxcode); + +// Read the next word from the bitstream "wvbits" and return the value. This +// function can be used for hybrid or lossless streams, but since an +// optimized version is available for lossless this function would normally +// be used for hybrid only. If a hybrid lossless stream is being read then +// the "correction" offset is written at the specified pointer. A return value +// of WORD_EOF indicates that the end of the bitstream was reached (all 1s) or +// some other error occurred. + +int32_t FASTCALL get_word(WavpackStream *wps, int chan, int32_t *correction) +{ + register struct entropy_data *c = wps->w.c + chan; + uint32_t ones_count, low, mid, high; + int32_t value; + int sign; + + if(!wps->wvbits.ptr) return WORD_EOF; + + if(correction) *correction = 0; + + if(!(wps->w.c[0].median[0] & ~1) && !wps->w.holding_zero && !wps->w.holding_one && !(wps->w.c[1].median[0] & ~1)) + { + uint32_t mask; + int cbits; + + if(wps->w.zeros_acc) + { + if(--wps->w.zeros_acc) + { + c->slow_level -= (c->slow_level + SLO) >> SLS; + return 0; + } + } + else + { + for(cbits = 0; cbits < 33 && getbit(&wps->wvbits); ++cbits); + + if(cbits == 33) return WORD_EOF; + + if(cbits < 2) + wps->w.zeros_acc = cbits; + else + { + for(mask = 1, wps->w.zeros_acc = 0; --cbits; mask <<= 1) + if(getbit(&wps->wvbits)) wps->w.zeros_acc |= mask; + + wps->w.zeros_acc |= mask; + } + + if(wps->w.zeros_acc) + { + c->slow_level -= (c->slow_level + SLO) >> SLS; + CLEAR(wps->w.c[0].median); + CLEAR(wps->w.c[1].median); + return 0; + } + } + } + + if(wps->w.holding_zero) + ones_count = wps->w.holding_zero = 0; + else + { +#ifdef USE_CTZ_OPTIMIZATION + while(wps->wvbits.bc < LIMIT_ONES) + { + if(++(wps->wvbits.ptr) == wps->wvbits.end) wps->wvbits.wrap(&wps->wvbits); + + wps->wvbits.sr |= *(wps->wvbits.ptr) << wps->wvbits.bc; + wps->wvbits.bc += sizeof(*(wps->wvbits.ptr)) * 8; + } + +#ifdef _WIN32 + _BitScanForward(&ones_count, ~wps->wvbits.sr); +#else + ones_count = __builtin_ctz(~wps->wvbits.sr); +#endif + + if(ones_count >= LIMIT_ONES) + { + wps->wvbits.bc -= ones_count; + wps->wvbits.sr >>= ones_count; + + for(; ones_count < (LIMIT_ONES + 1) && getbit(&wps->wvbits); ++ones_count); + + if(ones_count == (LIMIT_ONES + 1)) return WORD_EOF; + + if(ones_count == LIMIT_ONES) + { + uint32_t mask; + int cbits; + + for(cbits = 0; cbits < 33 && getbit(&wps->wvbits); ++cbits); + + if(cbits == 33) return WORD_EOF; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(&wps->wvbits)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } + } + else + { + wps->wvbits.bc -= ones_count + 1; + wps->wvbits.sr >>= ones_count + 1; + } +#elif defined(USE_NEXT8_OPTIMIZATION) + int next8; + + if(wps->wvbits.bc < 8) + { + if(++(wps->wvbits.ptr) == wps->wvbits.end) wps->wvbits.wrap(&wps->wvbits); + + next8 = (wps->wvbits.sr |= *(wps->wvbits.ptr) << wps->wvbits.bc) & 0xff; + wps->wvbits.bc += sizeof(*(wps->wvbits.ptr)) * 8; + } + else + next8 = wps->wvbits.sr & 0xff; + + if(next8 == 0xff) + { + wps->wvbits.bc -= 8; + wps->wvbits.sr >>= 8; + + for(ones_count = 8; ones_count < (LIMIT_ONES + 1) && getbit(&wps->wvbits); ++ones_count); + + if(ones_count == (LIMIT_ONES + 1)) return WORD_EOF; + + if(ones_count == LIMIT_ONES) + { + uint32_t mask; + int cbits; + + for(cbits = 0; cbits < 33 && getbit(&wps->wvbits); ++cbits); + + if(cbits == 33) return WORD_EOF; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(&wps->wvbits)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } + } + else + { + wps->wvbits.bc -= (ones_count = ones_count_table[next8]) + 1; + wps->wvbits.sr >>= ones_count + 1; + } +#else + for(ones_count = 0; ones_count < (LIMIT_ONES + 1) && getbit(&wps->wvbits); ++ones_count); + + if(ones_count >= LIMIT_ONES) + { + uint32_t mask; + int cbits; + + if(ones_count == (LIMIT_ONES + 1)) return WORD_EOF; + + for(cbits = 0; cbits < 33 && getbit(&wps->wvbits); ++cbits); + + if(cbits == 33) return WORD_EOF; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(&wps->wvbits)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } +#endif + + if(wps->w.holding_one) + { + wps->w.holding_one = ones_count & 1; + ones_count = (ones_count >> 1) + 1; + } + else + { + wps->w.holding_one = ones_count & 1; + ones_count >>= 1; + } + + wps->w.holding_zero = ~wps->w.holding_one & 1; + } + + if((wps->wphdr.flags & HYBRID_FLAG) && !chan) update_error_limit(wps); + + if(ones_count == 0) + { + low = 0; + high = GET_MED(0) - 1; + DEC_MED0(); + } + else + { + low = GET_MED(0); + INC_MED0(); + + if(ones_count == 1) + { + high = low + GET_MED(1) - 1; + DEC_MED1(); + } + else + { + low += GET_MED(1); + INC_MED1(); + + if(ones_count == 2) + { + high = low + GET_MED(2) - 1; + DEC_MED2(); + } + else + { + low += (ones_count - 2) * GET_MED(2); + high = low + GET_MED(2) - 1; + INC_MED2(); + } + } + } + + low &= 0x7fffffff; + high &= 0x7fffffff; + + if(low > high) // make sure high and low make sense + high = low; + + mid = (high + low + 1) >> 1; + + if(!c->error_limit) + mid = read_code(&wps->wvbits, high - low) + low; + else + while(high - low > c->error_limit) + { + if(getbit(&wps->wvbits)) + mid = (high + (low = mid) + 1) >> 1; + else + mid = ((high = mid - 1) + low + 1) >> 1; + } + + sign = getbit(&wps->wvbits); + + if(bs_is_open(&wps->wvcbits) && c->error_limit) + { + value = read_code(&wps->wvcbits, high - low) + low; + + if(correction) *correction = sign ? (mid - value) : (value - mid); + } + + if(wps->wphdr.flags & HYBRID_BITRATE) + { + c->slow_level -= (c->slow_level + SLO) >> SLS; + c->slow_level += wp_log2(mid); + } + + return sign ? ~mid : mid; +} + +// This is an optimized version of get_word() that is used for lossless only +// (error_limit == 0). Also, rather than obtaining a single sample, it can be +// used to obtain an entire buffer of either mono or stereo samples. + +int32_t get_words_lossless(WavpackStream *wps, int32_t *buffer, int32_t nsamples) +{ + struct entropy_data *c = wps->w.c; + uint32_t ones_count, low, high; + Bitstream *bs = &wps->wvbits; + int32_t csamples; +#ifdef USE_NEXT8_OPTIMIZATION + int32_t next8; +#endif + + if(nsamples && !bs->ptr) + { + memset(buffer, 0, (wps->wphdr.flags & MONO_DATA) ? nsamples * 4 : nsamples * 8); + return nsamples; + } + + if(!(wps->wphdr.flags & MONO_DATA)) nsamples *= 2; + + for(csamples = 0; csamples < nsamples; ++csamples) + { + if(!(wps->wphdr.flags & MONO_DATA)) c = wps->w.c + (csamples & 1); + + if(wps->w.holding_zero) + { + wps->w.holding_zero = 0; + low = read_code(bs, GET_MED(0) - 1); + DEC_MED0(); + buffer[csamples] = (getbit(bs)) ? ~low : low; + + if(++csamples == nsamples) break; + + if(!(wps->wphdr.flags & MONO_DATA)) c = wps->w.c + (csamples & 1); + } + + if(wps->w.c[0].median[0] < 2 && !wps->w.holding_one && wps->w.c[1].median[0] < 2) + { + uint32_t mask; + int cbits; + + if(wps->w.zeros_acc) + { + if(--wps->w.zeros_acc) + { + buffer[csamples] = 0; + continue; + } + } + else + { + for(cbits = 0; cbits < 33 && getbit(bs); ++cbits); + + if(cbits == 33) break; + + if(cbits < 2) + wps->w.zeros_acc = cbits; + else + { + for(mask = 1, wps->w.zeros_acc = 0; --cbits; mask <<= 1) + if(getbit(bs)) wps->w.zeros_acc |= mask; + + wps->w.zeros_acc |= mask; + } + + if(wps->w.zeros_acc) + { + CLEAR(wps->w.c[0].median); + CLEAR(wps->w.c[1].median); + buffer[csamples] = 0; + continue; + } + } + } + +#ifdef USE_CTZ_OPTIMIZATION + while(bs->bc < LIMIT_ONES) + { + if(++(bs->ptr) == bs->end) bs->wrap(bs); + + bs->sr |= *(bs->ptr) << bs->bc; + bs->bc += sizeof(*(bs->ptr)) * 8; + } + +#ifdef _WIN32 + _BitScanForward(&ones_count, ~wps->wvbits.sr); +#else + ones_count = __builtin_ctz(~wps->wvbits.sr); +#endif + + if(ones_count >= LIMIT_ONES) + { + bs->bc -= ones_count; + bs->sr >>= ones_count; + + for(; ones_count < (LIMIT_ONES + 1) && getbit(bs); ++ones_count); + + if(ones_count == (LIMIT_ONES + 1)) break; + + if(ones_count == LIMIT_ONES) + { + uint32_t mask; + int cbits; + + for(cbits = 0; cbits < 33 && getbit(bs); ++cbits); + + if(cbits == 33) break; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(bs)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } + } + else + { + bs->bc -= ones_count + 1; + bs->sr >>= ones_count + 1; + } +#elif defined(USE_NEXT8_OPTIMIZATION) + if(bs->bc < 8) + { + if(++(bs->ptr) == bs->end) bs->wrap(bs); + + next8 = (bs->sr |= *(bs->ptr) << bs->bc) & 0xff; + bs->bc += sizeof(*(bs->ptr)) * 8; + } + else + next8 = bs->sr & 0xff; + + if(next8 == 0xff) + { + bs->bc -= 8; + bs->sr >>= 8; + + for(ones_count = 8; ones_count < (LIMIT_ONES + 1) && getbit(bs); ++ones_count); + + if(ones_count == (LIMIT_ONES + 1)) break; + + if(ones_count == LIMIT_ONES) + { + uint32_t mask; + int cbits; + + for(cbits = 0; cbits < 33 && getbit(bs); ++cbits); + + if(cbits == 33) break; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(bs)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } + } + else + { + bs->bc -= (ones_count = ones_count_table[next8]) + 1; + bs->sr >>= ones_count + 1; + } +#else + for(ones_count = 0; ones_count < (LIMIT_ONES + 1) && getbit(bs); ++ones_count); + + if(ones_count >= LIMIT_ONES) + { + uint32_t mask; + int cbits; + + if(ones_count == (LIMIT_ONES + 1)) break; + + for(cbits = 0; cbits < 33 && getbit(bs); ++cbits); + + if(cbits == 33) break; + + if(cbits < 2) + ones_count = cbits; + else + { + for(mask = 1, ones_count = 0; --cbits; mask <<= 1) + if(getbit(bs)) ones_count |= mask; + + ones_count |= mask; + } + + ones_count += LIMIT_ONES; + } +#endif + + low = wps->w.holding_one; + wps->w.holding_one = ones_count & 1; + wps->w.holding_zero = ~ones_count & 1; + ones_count = (ones_count >> 1) + low; + + if(ones_count == 0) + { + low = 0; + high = GET_MED(0) - 1; + DEC_MED0(); + } + else + { + low = GET_MED(0); + INC_MED0(); + + if(ones_count == 1) + { + high = low + GET_MED(1) - 1; + DEC_MED1(); + } + else + { + low += GET_MED(1); + INC_MED1(); + + if(ones_count == 2) + { + high = low + GET_MED(2) - 1; + DEC_MED2(); + } + else + { + low += (ones_count - 2) * GET_MED(2); + high = low + GET_MED(2) - 1; + INC_MED2(); + } + } + } + + low += read_code(bs, high - low); + buffer[csamples] = (getbit(bs)) ? ~low : low; + } + + return (wps->wphdr.flags & MONO_DATA) ? csamples : (csamples / 2); +} + +// Read a single unsigned value from the specified bitstream with a value +// from 0 to maxcode. If there are exactly a power of two number of possible +// codes then this will read a fixed number of bits; otherwise it reads the +// minimum number of bits and then determines whether another bit is needed +// to define the code. + +static uint32_t __inline read_code(Bitstream *bs, uint32_t maxcode) +{ + unsigned long local_sr; + uint32_t extras, code; + int bitcount; + + if(maxcode < 2) return maxcode ? getbit(bs) : 0; + + bitcount = count_bits(maxcode); +#ifdef USE_BITMASK_TABLES + extras = bitset[bitcount] - maxcode - 1; +#else + extras = (1 << bitcount) - maxcode - 1; +#endif + + local_sr = bs->sr; + + while(bs->bc < bitcount) + { + if(++(bs->ptr) == bs->end) bs->wrap(bs); + + local_sr |= (long)*(bs->ptr) << bs->bc; + bs->bc += sizeof(*(bs->ptr)) * 8; + } + +#ifdef USE_BITMASK_TABLES + if((code = local_sr & bitmask[bitcount - 1]) >= extras) +#else + if((code = local_sr & ((1 << (bitcount - 1)) - 1)) >= extras) +#endif + code = (code << 1) - extras + ((local_sr >> (bitcount - 1)) & 1); + else + bitcount--; + + if(sizeof(local_sr) < 8 && bs->bc > sizeof(local_sr) * 8) + { + bs->bc -= bitcount; + bs->sr = *(bs->ptr) >> (sizeof(*(bs->ptr)) * 8 - bs->bc); + } + else + { + bs->bc -= bitcount; + bs->sr = local_sr >> bitcount; + } + + return code; +} diff --git a/wavpack/tags.c b/wavpack/tags.c new file mode 100644 index 0000000..d0100ad --- /dev/null +++ b/wavpack/tags.c @@ -0,0 +1,189 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// tags.c + +// This module provides support for reading metadata tags (either ID3v1 or +// APEv2) from WavPack files. No actual creation or manipulation of the tags +// is done in this module; this is just internal code to load the tags into +// memory. The high-level API functions are in the tag_utils.c module. + +#include +#include + +#include "wavpack_local.h" + +// This function attempts to load an ID3v1 or APEv2 tag from the specified +// file into the specified M_Tag structure. The ID3 tag fits in completely, +// but an APEv2 tag is variable length and so space must be allocated here +// to accomodate the data, and this will need to be freed later. A return +// value of TRUE indicates a valid tag was found and loaded. Note that the +// file pointer is undefined when this function exits. + +int load_tag(WavpackContext *wpc) +{ + int ape_tag_length, ape_tag_items; + M_Tag *m_tag = &wpc->m_tag; + + CLEAR(*m_tag); + + // This is a loop because we can try up to three times to look for an APEv2 tag. In order, we look: + // + // 1. At the end of the file for a APEv2 footer (this is the preferred location) + // 2. If there's instead an ID3v1 tag at the end of the file, try looking for an APEv2 footer right before that + // 3. If all else fails, look for an APEv2 header the the beginning of the file (use is strongly discouraged) + + while(1) + { + + // seek based on specific location that we are looking for tag (see above list) + + if(m_tag->tag_begins_file) // case #3 + wpc->reader->set_pos_abs(wpc->wv_in, 0); + else if(m_tag->id3_tag.tag_id[0] == 'T') // case #2 + wpc->reader->set_pos_rel(wpc->wv_in, -(int32_t)(sizeof(APE_Tag_Hdr) + sizeof(ID3_Tag)), SEEK_END); + else // case #1 + wpc->reader->set_pos_rel(wpc->wv_in, -(int32_t)sizeof(APE_Tag_Hdr), SEEK_END); + + // read a possible APEv2 tag header/footer and see if there's one there... + + if(wpc->reader->read_bytes(wpc->wv_in, &m_tag->ape_tag_hdr, sizeof(APE_Tag_Hdr)) == sizeof(APE_Tag_Hdr) && + !strncmp(m_tag->ape_tag_hdr.ID, "APETAGEX", 8)) + { + + WavpackLittleEndianToNative(&m_tag->ape_tag_hdr, APE_Tag_Hdr_Format); + + if(m_tag->ape_tag_hdr.version == 2000 && m_tag->ape_tag_hdr.item_count && + m_tag->ape_tag_hdr.length > sizeof(m_tag->ape_tag_hdr) && + m_tag->ape_tag_hdr.length <= APE_TAG_MAX_LENGTH && + (m_tag->ape_tag_data = malloc(m_tag->ape_tag_hdr.length)) != NULL) + { + + ape_tag_items = m_tag->ape_tag_hdr.item_count; + ape_tag_length = m_tag->ape_tag_hdr.length; + + // If this is a APEv2 footer (which is normal if we are searching at the end of the file)... + + if(!(m_tag->ape_tag_hdr.flags & APE_TAG_THIS_IS_HEADER)) + { + + if(m_tag->id3_tag.tag_id[0] == 'T') + m_tag->tag_file_pos = -(int32_t)sizeof(ID3_Tag); + else + m_tag->tag_file_pos = 0; + + m_tag->tag_file_pos -= ape_tag_length; + + // if the footer claims there is a header present also, we will read that and use it + // instead of the footer (after verifying it, of course) for enhanced robustness + + if(m_tag->ape_tag_hdr.flags & APE_TAG_CONTAINS_HEADER) m_tag->tag_file_pos -= sizeof(APE_Tag_Hdr); + + wpc->reader->set_pos_rel(wpc->wv_in, m_tag->tag_file_pos, SEEK_END); + + if(m_tag->ape_tag_hdr.flags & APE_TAG_CONTAINS_HEADER) + { + if(wpc->reader->read_bytes(wpc->wv_in, &m_tag->ape_tag_hdr, sizeof(APE_Tag_Hdr)) != + sizeof(APE_Tag_Hdr) || + strncmp(m_tag->ape_tag_hdr.ID, "APETAGEX", 8)) + { + free(m_tag->ape_tag_data); + CLEAR(*m_tag); + return FALSE; // something's wrong... + } + + WavpackLittleEndianToNative(&m_tag->ape_tag_hdr, APE_Tag_Hdr_Format); + + if(m_tag->ape_tag_hdr.version != 2000 || m_tag->ape_tag_hdr.item_count != ape_tag_items || + m_tag->ape_tag_hdr.length != ape_tag_length) + { + free(m_tag->ape_tag_data); + CLEAR(*m_tag); + return FALSE; // something's wrong... + } + } + } + + if(wpc->reader->read_bytes(wpc->wv_in, m_tag->ape_tag_data, ape_tag_length - sizeof(APE_Tag_Hdr)) != + ape_tag_length - sizeof(APE_Tag_Hdr)) + { + free(m_tag->ape_tag_data); + CLEAR(*m_tag); + return FALSE; // something's wrong... + } + else + { + CLEAR(m_tag->id3_tag); // ignore ID3v1 tag if we found APEv2 tag + return TRUE; + } + } + } + + // we come here if the search for the APEv2 tag failed (otherwise we would have returned with it) + + if(m_tag->id3_tag.tag_id[0] == 'T') + { // settle for the ID3v1 tag that we found + CLEAR(m_tag->ape_tag_hdr); + return TRUE; + } + + // if this was the search for the APEv2 tag at the beginning of the file (which is our + // last resort) then we have nothing, so return failure + + if(m_tag->tag_begins_file) + { + CLEAR(*m_tag); + return FALSE; + } + + // If we get here, then we have failed the first APEv2 tag search (at end of file) and so now we + // look for an ID3v1 tag at the same position. If that succeeds, then we'll loop back and look for + // an APEv2 tag immediately before the ID3v1 tag, otherwise our last resort is to look for an + // APEv2 tag at the beginning of the file. These are strongly discouraged (and not editable) but + // they have been seen in the wild so we attempt to handle them here (at least well enough to + // allow a proper transcoding). + + m_tag->tag_file_pos = -(int32_t)sizeof(ID3_Tag); + wpc->reader->set_pos_rel(wpc->wv_in, m_tag->tag_file_pos, SEEK_END); + + if(wpc->reader->read_bytes(wpc->wv_in, &m_tag->id3_tag, sizeof(ID3_Tag)) != sizeof(ID3_Tag) || + strncmp(m_tag->id3_tag.tag_id, "TAG", 3)) + { + m_tag->tag_begins_file = 1; // failed ID3v1, so look for APEv2 at beginning of file + CLEAR(m_tag->id3_tag); + } + } +} + +// Return TRUE is a valid ID3v1 or APEv2 tag has been loaded. + +int valid_tag(M_Tag *m_tag) +{ + if(m_tag->ape_tag_hdr.ID[0] == 'A') + return 'A'; + else if(m_tag->id3_tag.tag_id[0] == 'T') + return 'T'; + else + return 0; +} + +// Return FALSE if a valid APEv2 tag was only found at the beginning of the file (these are read-only +// because they cannot be edited without possibly shifting the entire file) + +int editable_tag(M_Tag *m_tag) { return !m_tag->tag_begins_file; } + +// Free the data for any APEv2 tag that was allocated. + +void free_tag(M_Tag *m_tag) +{ + if(m_tag->ape_tag_data) + { + free(m_tag->ape_tag_data); + m_tag->ape_tag_data = NULL; + } +} diff --git a/wavpack/unpack.c b/wavpack/unpack.c new file mode 100644 index 0000000..05e81bb --- /dev/null +++ b/wavpack/unpack.c @@ -0,0 +1,872 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// unpack.c + +// This module actually handles the decompression of the audio data, except for +// the entropy decoding which is handled by the read_words.c module. For better +// efficiency, the conversion is isolated to tight loops that handle an entire +// buffer. + +#include +#include + +#include "wavpack_local.h" + +#ifdef OPT_ASM_X86 +#define DECORR_STEREO_PASS_CONT unpack_decorr_stereo_pass_cont_x86 +#define DECORR_STEREO_PASS_CONT_AVAILABLE unpack_cpu_has_feature_x86(CPU_FEATURE_MMX) +#define DECORR_MONO_PASS_CONT unpack_decorr_mono_pass_cont_x86 +#elif defined(OPT_ASM_X64) && (defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW64__)) +#define DECORR_STEREO_PASS_CONT unpack_decorr_stereo_pass_cont_x64win +#define DECORR_STEREO_PASS_CONT_AVAILABLE 1 +#define DECORR_MONO_PASS_CONT unpack_decorr_mono_pass_cont_x64win +#elif defined(OPT_ASM_X64) +#define DECORR_STEREO_PASS_CONT unpack_decorr_stereo_pass_cont_x64 +#define DECORR_STEREO_PASS_CONT_AVAILABLE 1 +#define DECORR_MONO_PASS_CONT unpack_decorr_mono_pass_cont_x64 +#elif defined(OPT_ASM_ARM) +#define DECORR_STEREO_PASS_CONT unpack_decorr_stereo_pass_cont_armv7 +#define DECORR_STEREO_PASS_CONT_AVAILABLE 1 +#define DECORR_MONO_PASS_CONT unpack_decorr_mono_pass_cont_armv7 +#endif + +#ifdef DECORR_STEREO_PASS_CONT +extern void DECORR_STEREO_PASS_CONT(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count, int32_t long_math); +extern void DECORR_MONO_PASS_CONT(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count, int32_t long_math); +#endif + +// This flag provides the functionality of terminating the decoding and muting +// the output when a lossy sample appears to be corrupt. This is automatic +// for lossless files because a corrupt sample is unambigious, but for lossy +// data it might be possible for this to falsely trigger (although I have never +// seen it). + +#define LOSSY_MUTE + +///////////////////////////// executable code //////////////////////////////// + +// This monster actually unpacks the WavPack bitstream(s) into the specified +// buffer as 32-bit integers or floats (depending on orignal data). Lossy +// samples will be clipped to their original limits (i.e. 8-bit samples are +// clipped to -128/+127) but are still returned in longs. It is up to the +// caller to potentially reformat this for the final output including any +// multichannel distribution, block alignment or endian compensation. The +// function unpack_init() must have been called and the entire WavPack block +// must still be visible (although wps->blockbuff will not be accessed again). +// For maximum clarity, the function is broken up into segments that handle +// various modes. This makes for a few extra infrequent flag checks, but +// makes the code easier to follow because the nesting does not become so +// deep. For maximum efficiency, the conversion is isolated to tight loops +// that handle an entire buffer. The function returns the total number of +// samples unpacked, which can be less than the number requested if an error +// occurs or the end of the block is reached. + +static void decorr_stereo_pass(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count); +static void decorr_mono_pass(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count); +static void fixup_samples(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count); + +int32_t unpack_samples(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + uint32_t flags = wps->wphdr.flags, crc = wps->crc, i; + int32_t mute_limit = (1L << ((flags & MAG_MASK) >> MAG_LSB)) + 2; + int32_t correction[2], read_word, *bptr; + struct decorr_pass *dpp; + int tcount, m = 0; + + // don't attempt to decode past the end of the block, but watch out for overflow! + + if(wps->sample_index + sample_count > GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples && + GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples - wps->sample_index < sample_count) + sample_count = (uint32_t)(GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples - wps->sample_index); + + if(GET_BLOCK_INDEX(wps->wphdr) > wps->sample_index || wps->wphdr.block_samples < sample_count) + wps->mute_error = TRUE; + + if(wps->mute_error) + { + if(wpc->reduced_channels == 1 || wpc->config.num_channels == 1 || (flags & MONO_FLAG)) + memset(buffer, 0, sample_count * 4); + else + memset(buffer, 0, sample_count * 8); + + wps->sample_index += sample_count; + return sample_count; + } + + if((flags & HYBRID_FLAG) && !wps->block2buff) mute_limit = (mute_limit * 2) + 128; + + //////////////// handle lossless or hybrid lossy mono data ///////////////// + + if(!wps->block2buff && (flags & MONO_DATA)) + { + int32_t *eptr = buffer + sample_count; + + if(flags & HYBRID_FLAG) + { + i = sample_count; + + for(bptr = buffer; bptr < eptr;) + if((*bptr++ = get_word(wps, 0, NULL)) == WORD_EOF) + { + i = (uint32_t)(bptr - buffer); + break; + } + } + else + i = get_words_lossless(wps, buffer, sample_count); + +#ifdef DECORR_MONO_PASS_CONT + if(sample_count < 16) + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + decorr_mono_pass(dpp, buffer, sample_count); + else + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + int pre_samples = (dpp->term > MAX_TERM) ? 2 : dpp->term; + + decorr_mono_pass(dpp, buffer, pre_samples); + + DECORR_MONO_PASS_CONT(dpp, buffer + pre_samples, sample_count - pre_samples, + ((flags & MAG_MASK) >> MAG_LSB) > 15); + } +#else + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + decorr_mono_pass(dpp, buffer, sample_count); +#endif + +#ifndef LOSSY_MUTE + if(!(flags & HYBRID_FLAG)) +#endif + for(bptr = buffer; bptr < eptr; ++bptr) + { + if(labs(bptr[0]) > mute_limit) + { + i = (uint32_t)(bptr - buffer); + break; + } + + crc = crc * 3 + bptr[0]; + } +#ifndef LOSSY_MUTE + else + for(bptr = buffer; bptr < eptr; ++bptr) crc = crc * 3 + bptr[0]; +#endif + } + + /////////////// handle lossless or hybrid lossy stereo data /////////////// + + else if(!wps->block2buff && !(flags & MONO_DATA)) + { + int32_t *eptr = buffer + (sample_count * 2); + + if(flags & HYBRID_FLAG) + { + i = sample_count; + + for(bptr = buffer; bptr < eptr; bptr += 2) + if((bptr[0] = get_word(wps, 0, NULL)) == WORD_EOF || (bptr[1] = get_word(wps, 1, NULL)) == WORD_EOF) + { + i = (uint32_t)(bptr - buffer) / 2; + break; + } + } + else + i = get_words_lossless(wps, buffer, sample_count); + +#ifdef DECORR_STEREO_PASS_CONT + if(sample_count < 16 || !DECORR_STEREO_PASS_CONT_AVAILABLE) + { + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + decorr_stereo_pass(dpp, buffer, sample_count); + + m = sample_count & (MAX_TERM - 1); + } + else + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + int pre_samples = (dpp->term < 0 || dpp->term > MAX_TERM) ? 2 : dpp->term; + + decorr_stereo_pass(dpp, buffer, pre_samples); + + DECORR_STEREO_PASS_CONT(dpp, buffer + pre_samples * 2, sample_count - pre_samples, + ((flags & MAG_MASK) >> MAG_LSB) >= 16); + } +#else + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + decorr_stereo_pass(dpp, buffer, sample_count); + + m = sample_count & (MAX_TERM - 1); +#endif + + if(flags & JOINT_STEREO) + for(bptr = buffer; bptr < eptr; bptr += 2) + { + bptr[0] += (bptr[1] -= (bptr[0] >> 1)); + crc += (crc << 3) + (bptr[0] << 1) + bptr[0] + bptr[1]; + } + else + for(bptr = buffer; bptr < eptr; bptr += 2) crc += (crc << 3) + (bptr[0] << 1) + bptr[0] + bptr[1]; + +#ifndef LOSSY_MUTE + if(!(flags & HYBRID_FLAG)) +#endif + for(bptr = buffer; bptr < eptr; bptr += 16) + if(labs(bptr[0]) > mute_limit || labs(bptr[1]) > mute_limit) + { + i = (uint32_t)(bptr - buffer) / 2; + break; + } + } + + /////////////////// handle hybrid lossless mono data //////////////////// + + else if((flags & HYBRID_FLAG) && (flags & MONO_DATA)) + for(bptr = buffer, i = 0; i < sample_count; ++i) + { + + if((read_word = get_word(wps, 0, correction)) == WORD_EOF) break; + + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + int32_t sam, temp; + int k; + + if(dpp->term > MAX_TERM) + { + if(dpp->term & 1) + sam = 2 * dpp->samples_A[0] - dpp->samples_A[1]; + else + sam = (3 * dpp->samples_A[0] - dpp->samples_A[1]) >> 1; + + dpp->samples_A[1] = dpp->samples_A[0]; + k = 0; + } + else + { + sam = dpp->samples_A[m]; + k = (m + dpp->term) & (MAX_TERM - 1); + } + + temp = apply_weight(dpp->weight_A, sam) + read_word; + update_weight(dpp->weight_A, dpp->delta, sam, read_word); + dpp->samples_A[k] = read_word = temp; + } + + m = (m + 1) & (MAX_TERM - 1); + + if(flags & HYBRID_SHAPE) + { + int shaping_weight = (wps->dc.shaping_acc[0] += wps->dc.shaping_delta[0]) >> 16; + int32_t temp = -apply_weight(shaping_weight, wps->dc.error[0]); + + if((flags & NEW_SHAPING) && shaping_weight < 0 && temp) + { + if(temp == wps->dc.error[0]) temp = (temp < 0) ? temp + 1 : temp - 1; + + wps->dc.error[0] = temp - correction[0]; + } + else + wps->dc.error[0] = -correction[0]; + + read_word += correction[0] - temp; + } + else + read_word += correction[0]; + + crc += (crc << 1) + read_word; + + if(labs(read_word) > mute_limit) break; + + *bptr++ = read_word; + } + + //////////////////// handle hybrid lossless stereo data /////////////////// + + else if(wps->block2buff && !(flags & MONO_DATA)) + for(bptr = buffer, i = 0; i < sample_count; ++i) + { + int32_t left, right, left2, right2; + int32_t left_c = 0, right_c = 0; + + if((left = get_word(wps, 0, correction)) == WORD_EOF || + (right = get_word(wps, 1, correction + 1)) == WORD_EOF) + break; + + if(flags & CROSS_DECORR) + { + left_c = left + correction[0]; + right_c = right + correction[1]; + + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + int32_t sam_A, sam_B; + + if(dpp->term > 0) + { + if(dpp->term > MAX_TERM) + { + if(dpp->term & 1) + { + sam_A = 2 * dpp->samples_A[0] - dpp->samples_A[1]; + sam_B = 2 * dpp->samples_B[0] - dpp->samples_B[1]; + } + else + { + sam_A = (3 * dpp->samples_A[0] - dpp->samples_A[1]) >> 1; + sam_B = (3 * dpp->samples_B[0] - dpp->samples_B[1]) >> 1; + } + } + else + { + sam_A = dpp->samples_A[m]; + sam_B = dpp->samples_B[m]; + } + + left_c += apply_weight(dpp->weight_A, sam_A); + right_c += apply_weight(dpp->weight_B, sam_B); + } + else if(dpp->term == -1) + { + left_c += apply_weight(dpp->weight_A, dpp->samples_A[0]); + right_c += apply_weight(dpp->weight_B, left_c); + } + else + { + right_c += apply_weight(dpp->weight_B, dpp->samples_B[0]); + + if(dpp->term == -3) + left_c += apply_weight(dpp->weight_A, dpp->samples_A[0]); + else + left_c += apply_weight(dpp->weight_A, right_c); + } + } + + if(flags & JOINT_STEREO) left_c += (right_c -= (left_c >> 1)); + } + + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + { + int32_t sam_A, sam_B; + + if(dpp->term > 0) + { + int k; + + if(dpp->term > MAX_TERM) + { + if(dpp->term & 1) + { + sam_A = 2 * dpp->samples_A[0] - dpp->samples_A[1]; + sam_B = 2 * dpp->samples_B[0] - dpp->samples_B[1]; + } + else + { + sam_A = (3 * dpp->samples_A[0] - dpp->samples_A[1]) >> 1; + sam_B = (3 * dpp->samples_B[0] - dpp->samples_B[1]) >> 1; + } + + dpp->samples_A[1] = dpp->samples_A[0]; + dpp->samples_B[1] = dpp->samples_B[0]; + k = 0; + } + else + { + sam_A = dpp->samples_A[m]; + sam_B = dpp->samples_B[m]; + k = (m + dpp->term) & (MAX_TERM - 1); + } + + left2 = apply_weight(dpp->weight_A, sam_A) + left; + right2 = apply_weight(dpp->weight_B, sam_B) + right; + + update_weight(dpp->weight_A, dpp->delta, sam_A, left); + update_weight(dpp->weight_B, dpp->delta, sam_B, right); + + dpp->samples_A[k] = left = left2; + dpp->samples_B[k] = right = right2; + } + else if(dpp->term == -1) + { + left2 = left + apply_weight(dpp->weight_A, dpp->samples_A[0]); + update_weight_clip(dpp->weight_A, dpp->delta, dpp->samples_A[0], left); + left = left2; + right2 = right + apply_weight(dpp->weight_B, left2); + update_weight_clip(dpp->weight_B, dpp->delta, left2, right); + dpp->samples_A[0] = right = right2; + } + else + { + right2 = right + apply_weight(dpp->weight_B, dpp->samples_B[0]); + update_weight_clip(dpp->weight_B, dpp->delta, dpp->samples_B[0], right); + right = right2; + + if(dpp->term == -3) + { + right2 = dpp->samples_A[0]; + dpp->samples_A[0] = right; + } + + left2 = left + apply_weight(dpp->weight_A, right2); + update_weight_clip(dpp->weight_A, dpp->delta, right2, left); + dpp->samples_B[0] = left = left2; + } + } + + m = (m + 1) & (MAX_TERM - 1); + + if(!(flags & CROSS_DECORR)) + { + left_c = left + correction[0]; + right_c = right + correction[1]; + + if(flags & JOINT_STEREO) left_c += (right_c -= (left_c >> 1)); + } + + if(flags & JOINT_STEREO) left += (right -= (left >> 1)); + + if(flags & HYBRID_SHAPE) + { + int shaping_weight; + int32_t temp; + + correction[0] = left_c - left; + shaping_weight = (wps->dc.shaping_acc[0] += wps->dc.shaping_delta[0]) >> 16; + temp = -apply_weight(shaping_weight, wps->dc.error[0]); + + if((flags & NEW_SHAPING) && shaping_weight < 0 && temp) + { + if(temp == wps->dc.error[0]) temp = (temp < 0) ? temp + 1 : temp - 1; + + wps->dc.error[0] = temp - correction[0]; + } + else + wps->dc.error[0] = -correction[0]; + + left = left_c - temp; + correction[1] = right_c - right; + shaping_weight = (wps->dc.shaping_acc[1] += wps->dc.shaping_delta[1]) >> 16; + temp = -apply_weight(shaping_weight, wps->dc.error[1]); + + if((flags & NEW_SHAPING) && shaping_weight < 0 && temp) + { + if(temp == wps->dc.error[1]) temp = (temp < 0) ? temp + 1 : temp - 1; + + wps->dc.error[1] = temp - correction[1]; + } + else + wps->dc.error[1] = -correction[1]; + + right = right_c - temp; + } + else + { + left = left_c; + right = right_c; + } + + if(labs(left) > mute_limit || labs(right) > mute_limit) break; + + crc += (crc << 3) + (left << 1) + left + right; + *bptr++ = left; + *bptr++ = right; + } + else + i = 0; /* this line can't execute, but suppresses compiler warning */ + + if(i != sample_count) + { + memset(buffer, 0, sample_count * (flags & MONO_FLAG ? 4 : 8)); + wps->mute_error = TRUE; + i = sample_count; + + if(bs_is_open(&wps->wvxbits)) bs_close_read(&wps->wvxbits); + } + + if(m) + for(tcount = wps->num_terms, dpp = wps->decorr_passes; tcount--; dpp++) + if(dpp->term > 0 && dpp->term <= MAX_TERM) + { + int32_t temp_A[MAX_TERM], temp_B[MAX_TERM]; + int k; + + memcpy(temp_A, dpp->samples_A, sizeof(dpp->samples_A)); + memcpy(temp_B, dpp->samples_B, sizeof(dpp->samples_B)); + + for(k = 0; k < MAX_TERM; k++) + { + dpp->samples_A[k] = temp_A[m]; + dpp->samples_B[k] = temp_B[m]; + m = (m + 1) & (MAX_TERM - 1); + } + } + + fixup_samples(wpc, buffer, i); + + if((flags & FLOAT_DATA) && (wpc->open_flags & OPEN_NORMALIZE)) + WavpackFloatNormalize(buffer, (flags & MONO_DATA) ? i : i * 2, 127 - wps->float_norm_exp + wpc->norm_offset); + + if(flags & FALSE_STEREO) + { + int32_t *dptr = buffer + i * 2; + int32_t *sptr = buffer + i; + int32_t c = i; + + while(c--) + { + *--dptr = *--sptr; + *--dptr = *sptr; + } + } + + wps->sample_index += i; + wps->crc = crc; + + return i; +} + +// General function to perform mono decorrelation pass on specified buffer +// (although since this is the reverse function it might technically be called +// "correlation" instead). This version handles all sample resolutions and +// weight deltas. The dpp->samples_X[] data is returned normalized for term +// values 1-8. + +static void decorr_mono_pass(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count) +{ + int32_t delta = dpp->delta, weight_A = dpp->weight_A; + int32_t *bptr, *eptr = buffer + sample_count, sam_A; + int m, k; + + switch(dpp->term) + { + + case 17: + for(bptr = buffer; bptr < eptr; bptr++) + { + sam_A = 2 * dpp->samples_A[0] - dpp->samples_A[1]; + dpp->samples_A[1] = dpp->samples_A[0]; + dpp->samples_A[0] = apply_weight(weight_A, sam_A) + bptr[0]; + update_weight(weight_A, delta, sam_A, bptr[0]); + bptr[0] = dpp->samples_A[0]; + } + + break; + + case 18: + for(bptr = buffer; bptr < eptr; bptr++) + { + sam_A = (3 * dpp->samples_A[0] - dpp->samples_A[1]) >> 1; + dpp->samples_A[1] = dpp->samples_A[0]; + dpp->samples_A[0] = apply_weight(weight_A, sam_A) + bptr[0]; + update_weight(weight_A, delta, sam_A, bptr[0]); + bptr[0] = dpp->samples_A[0]; + } + + break; + + default: + for(m = 0, k = dpp->term & (MAX_TERM - 1), bptr = buffer; bptr < eptr; bptr++) + { + sam_A = dpp->samples_A[m]; + dpp->samples_A[k] = apply_weight(weight_A, sam_A) + bptr[0]; + update_weight(weight_A, delta, sam_A, bptr[0]); + bptr[0] = dpp->samples_A[k]; + m = (m + 1) & (MAX_TERM - 1); + k = (k + 1) & (MAX_TERM - 1); + } + + if(m) + { + int32_t temp_samples[MAX_TERM]; + + memcpy(temp_samples, dpp->samples_A, sizeof(dpp->samples_A)); + + for(k = 0; k < MAX_TERM; k++, m++) dpp->samples_A[k] = temp_samples[m & (MAX_TERM - 1)]; + } + + break; + } + + dpp->weight_A = weight_A; +} + +// General function to perform stereo decorrelation pass on specified buffer +// (although since this is the reverse function it might technically be called +// "correlation" instead). This version handles all sample resolutions and +// weight deltas. The dpp->samples_X[] data is *not* returned normalized for +// term values 1-8, so it should be normalized if it is going to be used to +// call this function again. + +static void decorr_stereo_pass(struct decorr_pass *dpp, int32_t *buffer, int32_t sample_count) +{ + int32_t *bptr, *eptr = buffer + (sample_count * 2); + int m, k; + + switch(dpp->term) + { + case 17: + for(bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam, tmp; + + sam = 2 * dpp->samples_A[0] - dpp->samples_A[1]; + dpp->samples_A[1] = dpp->samples_A[0]; + bptr[0] = dpp->samples_A[0] = apply_weight(dpp->weight_A, sam) + (tmp = bptr[0]); + update_weight(dpp->weight_A, dpp->delta, sam, tmp); + + sam = 2 * dpp->samples_B[0] - dpp->samples_B[1]; + dpp->samples_B[1] = dpp->samples_B[0]; + bptr[1] = dpp->samples_B[0] = apply_weight(dpp->weight_B, sam) + (tmp = bptr[1]); + update_weight(dpp->weight_B, dpp->delta, sam, tmp); + } + + break; + + case 18: + for(bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam, tmp; + + sam = dpp->samples_A[0] + ((dpp->samples_A[0] - dpp->samples_A[1]) >> 1); + dpp->samples_A[1] = dpp->samples_A[0]; + bptr[0] = dpp->samples_A[0] = apply_weight(dpp->weight_A, sam) + (tmp = bptr[0]); + update_weight(dpp->weight_A, dpp->delta, sam, tmp); + + sam = dpp->samples_B[0] + ((dpp->samples_B[0] - dpp->samples_B[1]) >> 1); + dpp->samples_B[1] = dpp->samples_B[0]; + bptr[1] = dpp->samples_B[0] = apply_weight(dpp->weight_B, sam) + (tmp = bptr[1]); + update_weight(dpp->weight_B, dpp->delta, sam, tmp); + } + + break; + + default: + for(m = 0, k = dpp->term & (MAX_TERM - 1), bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam; + + sam = dpp->samples_A[m]; + dpp->samples_A[k] = apply_weight(dpp->weight_A, sam) + bptr[0]; + update_weight(dpp->weight_A, dpp->delta, sam, bptr[0]); + bptr[0] = dpp->samples_A[k]; + + sam = dpp->samples_B[m]; + dpp->samples_B[k] = apply_weight(dpp->weight_B, sam) + bptr[1]; + update_weight(dpp->weight_B, dpp->delta, sam, bptr[1]); + bptr[1] = dpp->samples_B[k]; + + m = (m + 1) & (MAX_TERM - 1); + k = (k + 1) & (MAX_TERM - 1); + } + + break; + + case -1: + for(bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam; + + sam = bptr[0] + apply_weight(dpp->weight_A, dpp->samples_A[0]); + update_weight_clip(dpp->weight_A, dpp->delta, dpp->samples_A[0], bptr[0]); + bptr[0] = sam; + dpp->samples_A[0] = bptr[1] + apply_weight(dpp->weight_B, sam); + update_weight_clip(dpp->weight_B, dpp->delta, sam, bptr[1]); + bptr[1] = dpp->samples_A[0]; + } + + break; + + case -2: + for(bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam; + + sam = bptr[1] + apply_weight(dpp->weight_B, dpp->samples_B[0]); + update_weight_clip(dpp->weight_B, dpp->delta, dpp->samples_B[0], bptr[1]); + bptr[1] = sam; + dpp->samples_B[0] = bptr[0] + apply_weight(dpp->weight_A, sam); + update_weight_clip(dpp->weight_A, dpp->delta, sam, bptr[0]); + bptr[0] = dpp->samples_B[0]; + } + + break; + + case -3: + for(bptr = buffer; bptr < eptr; bptr += 2) + { + int32_t sam_A, sam_B; + + sam_A = bptr[0] + apply_weight(dpp->weight_A, dpp->samples_A[0]); + update_weight_clip(dpp->weight_A, dpp->delta, dpp->samples_A[0], bptr[0]); + sam_B = bptr[1] + apply_weight(dpp->weight_B, dpp->samples_B[0]); + update_weight_clip(dpp->weight_B, dpp->delta, dpp->samples_B[0], bptr[1]); + bptr[0] = dpp->samples_B[0] = sam_A; + bptr[1] = dpp->samples_A[0] = sam_B; + } + + break; + } +} + +// This is a helper function for unpack_samples() that applies several final +// operations. First, if the data is 32-bit float data, then that conversion +// is done in the float.c module (whether lossy or lossless) and we return. +// Otherwise, if the extended integer data applies, then that operation is +// executed first. If the unpacked data is lossy (and not corrected) then +// it is clipped and shifted in a single operation. Otherwise, if it's +// lossless then the last step is to apply the final shift (if any). + +static void fixup_samples(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + uint32_t flags = wps->wphdr.flags; + int lossy_flag = (flags & HYBRID_FLAG) && !wps->block2buff; + int shift = (flags & SHIFT_MASK) >> SHIFT_LSB; + + if(flags & FLOAT_DATA) + { + float_values(wps, buffer, (flags & MONO_DATA) ? sample_count : sample_count * 2); + return; + } + + if(flags & INT32_DATA) + { + uint32_t count = (flags & MONO_DATA) ? sample_count : sample_count * 2; + int sent_bits = wps->int32_sent_bits, zeros = wps->int32_zeros; + int ones = wps->int32_ones, dups = wps->int32_dups; + uint32_t data, mask = (1 << sent_bits) - 1; + int32_t *dptr = buffer; + + if(bs_is_open(&wps->wvxbits)) + { + uint32_t crc = wps->crc_x; + + while(count--) + { + // if (sent_bits) { + getbits(&data, sent_bits, &wps->wvxbits); + *dptr = (*dptr << sent_bits) | (data & mask); + // } + + if(zeros) + *dptr <<= zeros; + else if(ones) + *dptr = ((*dptr + 1) << ones) - 1; + else if(dups) + *dptr = ((*dptr + (*dptr & 1)) << dups) - (*dptr & 1); + + crc = crc * 9 + (*dptr & 0xffff) * 3 + ((*dptr >> 16) & 0xffff); + dptr++; + } + + wps->crc_x = crc; + } + else if(!sent_bits && (zeros + ones + dups)) + { + while(lossy_flag && (flags & BYTES_STORED) == 3 && shift < 8) + { + if(zeros) + zeros--; + else if(ones) + ones--; + else if(dups) + dups--; + else + break; + + shift++; + } + + while(count--) + { + if(zeros) + *dptr <<= zeros; + else if(ones) + *dptr = ((*dptr + 1) << ones) - 1; + else if(dups) + *dptr = ((*dptr + (*dptr & 1)) << dups) - (*dptr & 1); + + dptr++; + } + } + else + shift += zeros + sent_bits + ones + dups; + } + + if(lossy_flag) + { + int32_t min_value, max_value, min_shifted, max_shifted; + + switch(flags & BYTES_STORED) + { + case 0: + min_shifted = (min_value = -128 >> shift) << shift; + max_shifted = (max_value = 127 >> shift) << shift; + break; + + case 1: + min_shifted = (min_value = -32768 >> shift) << shift; + max_shifted = (max_value = 32767 >> shift) << shift; + break; + + case 2: + min_shifted = (min_value = -8388608 >> shift) << shift; + max_shifted = (max_value = 8388607 >> shift) << shift; + break; + + case 3: + default: /* "default" suppresses compiler warning */ + min_shifted = (min_value = (int32_t)0x80000000 >> shift) << shift; + max_shifted = (max_value = (int32_t)0x7fffffff >> shift) << shift; + break; + } + + if(!(flags & MONO_DATA)) sample_count *= 2; + + while(sample_count--) + { + if(*buffer < min_value) + *buffer++ = min_shifted; + else if(*buffer > max_value) + *buffer++ = max_shifted; + else + *buffer++ <<= shift; + } + } + else if(shift) + { + if(!(flags & MONO_DATA)) sample_count *= 2; + + while(sample_count--) *buffer++ <<= shift; + } +} + +// This function checks the crc value(s) for an unpacked block, returning the +// number of actual crc errors detected for the block. The block must be +// completely unpacked before this test is valid. For losslessly unpacked +// blocks of float or extended integer data the extended crc is also checked. +// Note that WavPack's crc is not a CCITT approved polynomial algorithm, but +// is a much simpler method that is virtually as robust for real world data. + +int check_crc_error(WavpackContext *wpc) +{ + int result = 0, stream; + + for(stream = 0; stream < wpc->num_streams; stream++) + { + WavpackStream *wps = wpc->streams[stream]; + + if(wps->crc != wps->wphdr.crc) + ++result; + else if(bs_is_open(&wps->wvxbits) && wps->crc_x != wps->crc_wvx) + ++result; + } + + return result; +} diff --git a/wavpack/unpack_floats.c b/wavpack/unpack_floats.c new file mode 100644 index 0000000..70d666b --- /dev/null +++ b/wavpack/unpack_floats.c @@ -0,0 +1,154 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// unpack_floats.c + +// This module deals with the restoration of floating-point data. Note that no +// floating point math is involved here...the values are only processed with +// the macros that directly access the mantissa, exponent, and sign fields. +// That's why we use the f32 type instead of the built-in float type. + +#include + +#include "wavpack_local.h" + +static void float_values_nowvx(WavpackStream *wps, int32_t *values, int32_t num_values); + +void float_values(WavpackStream *wps, int32_t *values, int32_t num_values) +{ + uint32_t crc = wps->crc_x; + + if(!bs_is_open(&wps->wvxbits)) + { + float_values_nowvx(wps, values, num_values); + return; + } + + while(num_values--) + { + int shift_count = 0, exp = wps->float_max_exp; + f32 outval = 0; + uint32_t temp; + + if(*values == 0) + { + if(wps->float_flags & FLOAT_ZEROS_SENT) + { + if(getbit(&wps->wvxbits)) + { + getbits(&temp, 23, &wps->wvxbits); + set_mantissa(outval, temp); + + if(exp >= 25) + { + getbits(&temp, 8, &wps->wvxbits); + set_exponent(outval, temp); + } + + set_sign(outval, getbit(&wps->wvxbits)); + } + else if(wps->float_flags & FLOAT_NEG_ZEROS) + set_sign(outval, getbit(&wps->wvxbits)); + } + } + else + { + *values <<= wps->float_shift; + + if(*values < 0) + { + *values = -*values; + set_sign(outval, 1); + } + + if(*values == 0x1000000) + { + if(getbit(&wps->wvxbits)) + { + getbits(&temp, 23, &wps->wvxbits); + set_mantissa(outval, temp); + } + + set_exponent(outval, 255); + } + else + { + if(exp) + while(!(*values & 0x800000) && --exp) + { + shift_count++; + *values <<= 1; + } + + if(shift_count) + { + if((wps->float_flags & FLOAT_SHIFT_ONES) || + ((wps->float_flags & FLOAT_SHIFT_SAME) && getbit(&wps->wvxbits))) + *values |= ((1 << shift_count) - 1); + else if(wps->float_flags & FLOAT_SHIFT_SENT) + { + getbits(&temp, shift_count, &wps->wvxbits); + *values |= temp & ((1 << shift_count) - 1); + } + } + + set_mantissa(outval, *values); + set_exponent(outval, exp); + } + } + + crc = crc * 27 + get_mantissa(outval) * 9 + get_exponent(outval) * 3 + get_sign(outval); + *(f32 *)values++ = outval; + } + + wps->crc_x = crc; +} + +static void float_values_nowvx(WavpackStream *wps, int32_t *values, int32_t num_values) +{ + while(num_values--) + { + int shift_count = 0, exp = wps->float_max_exp; + f32 outval = 0; + + if(*values) + { + *values <<= wps->float_shift; + + if(*values < 0) + { + *values = -*values; + set_sign(outval, 1); + } + + if(*values >= 0x1000000) + { + while(*values & 0xf000000) + { + *values >>= 1; + ++exp; + } + } + else if(exp) + { + while(!(*values & 0x800000) && --exp) + { + shift_count++; + *values <<= 1; + } + + if(shift_count && (wps->float_flags & FLOAT_SHIFT_ONES)) *values |= ((1 << shift_count) - 1); + } + + set_mantissa(outval, *values); + set_exponent(outval, exp); + } + + *(f32 *)values++ = outval; + } +} diff --git a/wavpack/unpack_seek.c b/wavpack/unpack_seek.c new file mode 100644 index 0000000..82e9ef3 --- /dev/null +++ b/wavpack/unpack_seek.c @@ -0,0 +1,404 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// unpack_seek.c + +// This module provides the high-level API for unpacking audio data from +// a specific sample index (i.e., seeking). + +#ifndef NO_SEEKING + +#include +#include + +#include "wavpack_local.h" + +///////////////////////////// executable code //////////////////////////////// + +static int64_t find_sample(WavpackContext *wpc, void *infile, int64_t header_pos, int64_t sample); + +// Seek to the specifed sample index, returning TRUE on success. Note that +// files generated with version 4.0 or newer will seek almost immediately. +// Older files can take quite long if required to seek through unplayed +// portions of the file, but will create a seek map so that reverse seeks +// (or forward seeks to already scanned areas) will be very fast. After a +// FALSE return the file should not be accessed again (other than to close +// it); this is a fatal error. + +int WavpackSeekSample(WavpackContext *wpc, uint32_t sample) { return WavpackSeekSample64(wpc, sample); } + +int WavpackSeekSample64(WavpackContext *wpc, int64_t sample) +{ + WavpackStream *wps = wpc->streams ? wpc->streams[wpc->current_stream = 0] : NULL; + uint32_t bcount, samples_to_skip, samples_to_decode = 0; + int32_t *buffer; + + if(wpc->total_samples == -1 || sample >= wpc->total_samples || !wpc->reader->can_seek(wpc->wv_in) || + (wpc->open_flags & OPEN_STREAMING) || (wpc->wvc_flag && !wpc->reader->can_seek(wpc->wvc_in))) + return FALSE; + +#ifdef ENABLE_LEGACY + if(wpc->stream3) return seek_sample3(wpc, (uint32_t)sample); +#endif + +#ifdef ENABLE_DSD + if(wpc->decimation_context) + { // the decimation code needs some context to be sample accurate + if(sample < 16) + { + samples_to_decode = (uint32_t)sample; + sample = 0; + } + else + { + samples_to_decode = 16; + sample -= 16; + } + } +#endif + + if(!wps->wphdr.block_samples || !(wps->wphdr.flags & INITIAL_BLOCK) || sample < GET_BLOCK_INDEX(wps->wphdr) || + sample >= GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples) + { + + free_streams(wpc); + wpc->filepos = find_sample(wpc, wpc->wv_in, wpc->filepos, sample); + + if(wpc->filepos == -1) return FALSE; + + if(wpc->wvc_flag) + { + wpc->file2pos = find_sample(wpc, wpc->wvc_in, 0, sample); + + if(wpc->file2pos == -1) return FALSE; + } + } + + if(!wps->blockbuff) + { + wpc->reader->set_pos_abs(wpc->wv_in, wpc->filepos); + wpc->reader->read_bytes(wpc->wv_in, &wps->wphdr, sizeof(WavpackHeader)); + WavpackLittleEndianToNative(&wps->wphdr, WavpackHeaderFormat); + wps->blockbuff = malloc(wps->wphdr.ckSize + 8); + memcpy(wps->blockbuff, &wps->wphdr, sizeof(WavpackHeader)); + + if(wpc->reader->read_bytes(wpc->wv_in, wps->blockbuff + sizeof(WavpackHeader), wps->wphdr.ckSize - 24) != + wps->wphdr.ckSize - 24) + { + free_streams(wpc); + return FALSE; + } + + // render corrupt blocks harmless + if(!WavpackVerifySingleBlock(wps->blockbuff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.ckSize = sizeof(WavpackHeader) - 8; + wps->wphdr.block_samples = 0; + memcpy(wps->blockbuff, &wps->wphdr, 32); + } + + SET_BLOCK_INDEX(wps->wphdr, GET_BLOCK_INDEX(wps->wphdr) - wpc->initial_index); + memcpy(wps->blockbuff, &wps->wphdr, sizeof(WavpackHeader)); + wps->init_done = FALSE; + + if(wpc->wvc_flag) + { + wpc->reader->set_pos_abs(wpc->wvc_in, wpc->file2pos); + wpc->reader->read_bytes(wpc->wvc_in, &wps->wphdr, sizeof(WavpackHeader)); + WavpackLittleEndianToNative(&wps->wphdr, WavpackHeaderFormat); + wps->block2buff = malloc(wps->wphdr.ckSize + 8); + memcpy(wps->block2buff, &wps->wphdr, sizeof(WavpackHeader)); + + if(wpc->reader->read_bytes(wpc->wvc_in, wps->block2buff + sizeof(WavpackHeader), wps->wphdr.ckSize - 24) != + wps->wphdr.ckSize - 24) + { + free_streams(wpc); + return FALSE; + } + + // render corrupt blocks harmless + if(!WavpackVerifySingleBlock(wps->block2buff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.ckSize = sizeof(WavpackHeader) - 8; + wps->wphdr.block_samples = 0; + memcpy(wps->block2buff, &wps->wphdr, 32); + } + + SET_BLOCK_INDEX(wps->wphdr, GET_BLOCK_INDEX(wps->wphdr) - wpc->initial_index); + memcpy(wps->block2buff, &wps->wphdr, sizeof(WavpackHeader)); + } + + if(!wps->init_done && !unpack_init(wpc)) + { + free_streams(wpc); + return FALSE; + } + + wps->init_done = TRUE; + } + + while(!wpc->reduced_channels && !(wps->wphdr.flags & FINAL_BLOCK)) + { + if(++wpc->current_stream == wpc->num_streams) + { + + if(wpc->num_streams == wpc->max_streams) + { + free_streams(wpc); + return FALSE; + } + + wpc->streams = realloc(wpc->streams, (wpc->num_streams + 1) * sizeof(wpc->streams[0])); + wps = wpc->streams[wpc->num_streams++] = malloc(sizeof(WavpackStream)); + CLEAR(*wps); + bcount = read_next_header(wpc->reader, wpc->wv_in, &wps->wphdr); + + if(bcount == (uint32_t)-1) + { + free_streams(wpc); + return FALSE; + } + + wps->blockbuff = malloc(wps->wphdr.ckSize + 8); + memcpy(wps->blockbuff, &wps->wphdr, 32); + + if(wpc->reader->read_bytes(wpc->wv_in, wps->blockbuff + 32, wps->wphdr.ckSize - 24) != + wps->wphdr.ckSize - 24) + { + free_streams(wpc); + return FALSE; + } + + // render corrupt blocks harmless + if(!WavpackVerifySingleBlock(wps->blockbuff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.ckSize = sizeof(WavpackHeader) - 8; + wps->wphdr.block_samples = 0; + memcpy(wps->blockbuff, &wps->wphdr, 32); + } + + wps->init_done = FALSE; + + if(wpc->wvc_flag && !read_wvc_block(wpc)) + { + free_streams(wpc); + return FALSE; + } + + if(!wps->init_done && !unpack_init(wpc)) + { + free_streams(wpc); + return FALSE; + } + + wps->init_done = TRUE; + } + else + wps = wpc->streams[wpc->current_stream]; + } + + if(sample < wps->sample_index) + { + for(wpc->current_stream = 0; wpc->current_stream < wpc->num_streams; wpc->current_stream++) + if(!unpack_init(wpc)) + return FALSE; + else + wpc->streams[wpc->current_stream]->init_done = TRUE; + } + + samples_to_skip = (uint32_t)(sample - wps->sample_index); + + if(samples_to_skip > 131072) + { + free_streams(wpc); + return FALSE; + } + + if(samples_to_skip) + { + buffer = malloc(samples_to_skip * 8); + + for(wpc->current_stream = 0; wpc->current_stream < wpc->num_streams; wpc->current_stream++) +#ifdef ENABLE_DSD + if(wpc->streams[wpc->current_stream]->wphdr.flags & DSD_FLAG) + unpack_dsd_samples(wpc, buffer, samples_to_skip); + else +#endif + unpack_samples(wpc, buffer, samples_to_skip); + + free(buffer); + } + + wpc->current_stream = 0; + +#ifdef ENABLE_DSD + if(wpc->decimation_context) decimate_dsd_reset(wpc->decimation_context); + + if(samples_to_decode) + { + buffer = malloc(samples_to_decode * wpc->config.num_channels * 4); + + if(buffer) + { + WavpackUnpackSamples(wpc, buffer, samples_to_decode); + free(buffer); + } + } +#endif + + return TRUE; +} + +// Find a valid WavPack header, searching either from the current file position +// (or from the specified position if not -1) and store it (endian corrected) +// at the specified pointer. The return value is the exact file position of the +// header, although we may have actually read past it. Because this function +// is used for seeking to a specific audio sample, it only considers blocks +// that contain audio samples for the initial stream to be valid. + +#define BUFSIZE 4096 + +static int64_t find_header(WavpackStreamReader64 *reader, void *id, int64_t filepos, WavpackHeader *wphdr) +{ + unsigned char *buffer = malloc(BUFSIZE), *sp = buffer, *ep = buffer; + + if(filepos != (uint32_t)-1 && reader->set_pos_abs(id, filepos)) + { + free(buffer); + return -1; + } + + while(1) + { + int bleft; + + if(sp < ep) + { + bleft = (int)(ep - sp); + memcpy(buffer, sp, bleft); + ep -= (sp - buffer); + sp = buffer; + } + else + { + if(sp > ep) + if(reader->set_pos_rel(id, (int32_t)(sp - ep), SEEK_CUR)) + { + free(buffer); + return -1; + } + + sp = ep = buffer; + bleft = 0; + } + + ep += reader->read_bytes(id, ep, BUFSIZE - bleft); + + if(ep - sp < 32) + { + free(buffer); + return -1; + } + + while(sp + 32 <= ep) + if(*sp++ == 'w' && *sp == 'v' && *++sp == 'p' && *++sp == 'k' && !(*++sp & 1) && sp[2] < 16 && !sp[3] && + (sp[2] || sp[1] || *sp >= 24) && sp[5] == 4 && sp[4] >= (MIN_STREAM_VERS & 0xff) && + sp[4] <= (MAX_STREAM_VERS & 0xff) && sp[18] < 3 && !sp[19]) + { + memcpy(wphdr, sp - 4, sizeof(*wphdr)); + WavpackLittleEndianToNative(wphdr, WavpackHeaderFormat); + + if(wphdr->block_samples && (wphdr->flags & INITIAL_BLOCK)) + { + free(buffer); + return reader->get_pos(id) - (ep - sp + 4); + } + + if(wphdr->ckSize > 1024) sp += wphdr->ckSize - 1024; + } + } +} + +// Find the WavPack block that contains the specified sample. If "header_pos" +// is zero, then no information is assumed except the total number of samples +// in the file and its size in bytes. If "header_pos" is non-zero then we +// assume that it is the file position of the valid header image contained in +// the first stream and we can limit our search to either the portion above +// or below that point. If a .wvc file is being used, then this must be called +// for that file also. + +static int64_t find_sample(WavpackContext *wpc, void *infile, int64_t header_pos, int64_t sample) +{ + WavpackStream *wps = wpc->streams[wpc->current_stream]; + int64_t file_pos1 = 0, file_pos2 = wpc->reader->get_length(infile); + int64_t sample_pos1 = 0, sample_pos2 = wpc->total_samples; + double ratio = 0.96; + int file_skip = 0; + + if(sample >= wpc->total_samples) return -1; + + if(header_pos && wps->wphdr.block_samples) + { + if(GET_BLOCK_INDEX(wps->wphdr) > sample) + { + sample_pos2 = GET_BLOCK_INDEX(wps->wphdr); + file_pos2 = header_pos; + } + else if(GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples <= sample) + { + sample_pos1 = GET_BLOCK_INDEX(wps->wphdr); + file_pos1 = header_pos; + } + else + return header_pos; + } + + while(1) + { + double bytes_per_sample; + int64_t seek_pos; + + bytes_per_sample = (double)file_pos2 - file_pos1; + bytes_per_sample /= sample_pos2 - sample_pos1; + seek_pos = file_pos1 + (file_skip ? 32 : 0); + seek_pos += (int64_t)(bytes_per_sample * (sample - sample_pos1) * ratio); + seek_pos = find_header(wpc->reader, infile, seek_pos, &wps->wphdr); + + if(seek_pos != (int64_t)-1) SET_BLOCK_INDEX(wps->wphdr, GET_BLOCK_INDEX(wps->wphdr) - wpc->initial_index); + + if(seek_pos == (int64_t)-1 || seek_pos >= file_pos2) + { + if(ratio > 0.0) + { + if((ratio -= 0.24) < 0.0) ratio = 0.0; + } + else + return -1; + } + else if(GET_BLOCK_INDEX(wps->wphdr) > sample) + { + sample_pos2 = GET_BLOCK_INDEX(wps->wphdr); + file_pos2 = seek_pos; + } + else if(GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples <= sample) + { + + if(seek_pos == file_pos1) + file_skip = 1; + else + { + sample_pos1 = GET_BLOCK_INDEX(wps->wphdr); + file_pos1 = seek_pos; + } + } + else + return seek_pos; + } +} + +#endif diff --git a/wavpack/unpack_utils.c b/wavpack/unpack_utils.c new file mode 100644 index 0000000..551f0ce --- /dev/null +++ b/wavpack/unpack_utils.c @@ -0,0 +1,417 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// unpack_utils.c + +// This module provides the high-level API for unpacking audio data from +// WavPack files. It manages the buffers used to interleave the data passed +// back to the application from the individual streams. The actual audio +// stream decompression is handled in the unpack.c module. + +#include +#include + +#include "wavpack_local.h" + +///////////////////////////// executable code //////////////////////////////// + +// Unpack the specified number of samples from the current file position. +// Note that "samples" here refers to "complete" samples, which would be +// 2 longs for stereo files or even more for multichannel files, so the +// required memory at "buffer" is 4 * samples * num_channels bytes. The +// audio data is returned right-justified in 32-bit longs in the endian +// mode native to the executing processor. So, if the original data was +// 16-bit, then the values returned would be +/-32k. Floating point data +// can also be returned if the source was floating point data (and this +// can be optionally normalized to +/-1.0 by using the appropriate flag +// in the call to WavpackOpenFileInput ()). The actual number of samples +// unpacked is returned, which should be equal to the number requested unless +// the end of fle is encountered or an error occurs. After all samples have +// been unpacked then 0 will be returned. + +uint32_t WavpackUnpackSamples(WavpackContext *wpc, int32_t *buffer, uint32_t samples) +{ + WavpackStream *wps = wpc->streams ? wpc->streams[wpc->current_stream = 0] : NULL; + int num_channels = wpc->config.num_channels, file_done = FALSE; + uint32_t bcount, samples_unpacked = 0, samples_to_unpack; + int32_t *bptr = buffer; + +#ifdef ENABLE_LEGACY + if(wpc->stream3) return unpack_samples3(wpc, buffer, samples); +#endif + + while(samples) + { + + // if the current block has no audio, or it's not the first block of a multichannel + // sequence, or the sample we're on is past the last sample in this block...we need + // to free up the streams and read the next block + + if(!wps->wphdr.block_samples || !(wps->wphdr.flags & INITIAL_BLOCK) || + wps->sample_index >= GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples) + { + + int64_t nexthdrpos; + + if(wpc->wrapper_bytes >= MAX_WRAPPER_BYTES) break; + + free_streams(wpc); + nexthdrpos = wpc->reader->get_pos(wpc->wv_in); + bcount = read_next_header(wpc->reader, wpc->wv_in, &wps->wphdr); + + if(bcount == (uint32_t)-1) break; + + wpc->filepos = nexthdrpos + bcount; + + // allocate the memory for the entire raw block and read it in + + wps->blockbuff = malloc(wps->wphdr.ckSize + 8); + + if(!wps->blockbuff) break; + + memcpy(wps->blockbuff, &wps->wphdr, 32); + + if(wpc->reader->read_bytes(wpc->wv_in, wps->blockbuff + 32, wps->wphdr.ckSize - 24) != + wps->wphdr.ckSize - 24) + { + strcpy(wpc->error_message, "can't read all of last block!"); + wps->wphdr.block_samples = 0; + wps->wphdr.ckSize = 24; + break; + } + + // render corrupt blocks harmless + if(!WavpackVerifySingleBlock(wps->blockbuff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.ckSize = sizeof(WavpackHeader) - 8; + wps->wphdr.block_samples = 0; + memcpy(wps->blockbuff, &wps->wphdr, 32); + } + + // potentially adjusting block_index must be done AFTER verifying block + + if(wpc->open_flags & OPEN_STREAMING) + SET_BLOCK_INDEX(wps->wphdr, wps->sample_index = 0); + else + SET_BLOCK_INDEX(wps->wphdr, GET_BLOCK_INDEX(wps->wphdr) - wpc->initial_index); + + memcpy(wps->blockbuff, &wps->wphdr, 32); + wps->init_done = FALSE; // we have not yet called unpack_init() for this block + + // if this block has audio, but not the sample index we were expecting, flag an error + + if(wps->wphdr.block_samples && wps->sample_index != GET_BLOCK_INDEX(wps->wphdr)) wpc->crc_errors++; + + // if this block has audio, and we're in hybrid lossless mode, read the matching wvc block + + if(wps->wphdr.block_samples && wpc->wvc_flag) read_wvc_block(wpc); + + // if the block does NOT have any audio, call unpack_init() to process non-audio stuff + + if(!wps->wphdr.block_samples) + { + if(!wps->init_done && !unpack_init(wpc)) wpc->crc_errors++; + + wps->init_done = TRUE; + } + } + + // if the current block has no audio, or it's not the first block of a multichannel + // sequence, or the sample we're on is past the last sample in this block...we need + // to loop back and read the next block + + if(!wps->wphdr.block_samples || !(wps->wphdr.flags & INITIAL_BLOCK) || + wps->sample_index >= GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples) + continue; + + // There seems to be some missing data, like a block was corrupted or something. + // If it's not too much data, just fill in with silence here and loop back. + + if(wps->sample_index < GET_BLOCK_INDEX(wps->wphdr)) + { + int32_t zvalue = (wps->wphdr.flags & DSD_FLAG) ? 0x55 : 0; + + samples_to_unpack = (uint32_t)(GET_BLOCK_INDEX(wps->wphdr) - wps->sample_index); + + if(!samples_to_unpack || samples_to_unpack > 262144) + { + strcpy(wpc->error_message, "discontinuity found, aborting file!"); + wps->wphdr.block_samples = 0; + wps->wphdr.ckSize = 24; + break; + } + + if(samples_to_unpack > samples) samples_to_unpack = samples; + + wps->sample_index += samples_to_unpack; + samples_unpacked += samples_to_unpack; + samples -= samples_to_unpack; + + samples_to_unpack *= (wpc->reduced_channels ? wpc->reduced_channels : num_channels); + + while(samples_to_unpack--) *bptr++ = zvalue; + + continue; + } + + // calculate number of samples to process from this block, then initialize the decoder for + // this block if we haven't already + + samples_to_unpack = (uint32_t)(GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples - wps->sample_index); + + if(samples_to_unpack > samples) samples_to_unpack = samples; + + if(!wps->init_done && !unpack_init(wpc)) wpc->crc_errors++; + + wps->init_done = TRUE; + + // if this block is not the final block of a multichannel sequence (and we're not truncating + // to stereo), then enter this conditional block...otherwise we just unpack the samples directly + + if(!wpc->reduced_channels && !(wps->wphdr.flags & FINAL_BLOCK)) + { + int32_t *temp_buffer = malloc(samples_to_unpack * 8), *src, *dst; + int offset = 0; // offset to next channel in sequence (0 to num_channels - 1) + uint32_t samcnt; + + // since we are getting samples from multiple bocks in a multichannel sequence, we must + // allocate a temporary buffer to unpack to so that we can re-interleave the samples + + if(!temp_buffer) break; + + // loop through all the streams... + + while(1) + { + + // if the stream has not been allocated and corresponding block read, do that here... + + if(wpc->current_stream == wpc->num_streams) + { + wpc->streams = realloc(wpc->streams, (wpc->num_streams + 1) * sizeof(wpc->streams[0])); + + if(!wpc->streams) break; + + wps = wpc->streams[wpc->num_streams++] = malloc(sizeof(WavpackStream)); + + if(!wps) break; + + CLEAR(*wps); + bcount = read_next_header(wpc->reader, wpc->wv_in, &wps->wphdr); + + if(bcount == (uint32_t)-1) + { + wpc->streams[0]->wphdr.block_samples = 0; + wpc->streams[0]->wphdr.ckSize = 24; + file_done = TRUE; + break; + } + + wps->blockbuff = malloc(wps->wphdr.ckSize + 8); + + if(!wps->blockbuff) break; + + memcpy(wps->blockbuff, &wps->wphdr, 32); + + if(wpc->reader->read_bytes(wpc->wv_in, wps->blockbuff + 32, wps->wphdr.ckSize - 24) != + wps->wphdr.ckSize - 24) + { + wpc->streams[0]->wphdr.block_samples = 0; + wpc->streams[0]->wphdr.ckSize = 24; + file_done = TRUE; + break; + } + + // render corrupt blocks harmless + if(!WavpackVerifySingleBlock(wps->blockbuff, !(wpc->open_flags & OPEN_NO_CHECKSUM))) + { + wps->wphdr.ckSize = sizeof(WavpackHeader) - 8; + wps->wphdr.block_samples = 0; + memcpy(wps->blockbuff, &wps->wphdr, 32); + } + + // potentially adjusting block_index must be done AFTER verifying block + + if(wpc->open_flags & OPEN_STREAMING) + SET_BLOCK_INDEX(wps->wphdr, wps->sample_index = 0); + else + SET_BLOCK_INDEX(wps->wphdr, GET_BLOCK_INDEX(wps->wphdr) - wpc->initial_index); + + memcpy(wps->blockbuff, &wps->wphdr, 32); + + // if this block has audio, and we're in hybrid lossless mode, read the matching wvc block + + if(wpc->wvc_flag) read_wvc_block(wpc); + + // initialize the unpacker for this block + + if(!unpack_init(wpc)) wpc->crc_errors++; + + wps->init_done = TRUE; + } + else + wps = wpc->streams[wpc->current_stream]; + + // unpack the correct number of samples (either mono or stereo) into the temp buffer + +#ifdef ENABLE_DSD + if(wps->wphdr.flags & DSD_FLAG) + unpack_dsd_samples(wpc, src = temp_buffer, samples_to_unpack); + else +#endif + unpack_samples(wpc, src = temp_buffer, samples_to_unpack); + + samcnt = samples_to_unpack; + dst = bptr + offset; + + // if the block is mono, copy the samples from the single channel into the destination + // using num_channels as the stride + + if(wps->wphdr.flags & MONO_FLAG) + { + while(samcnt--) + { + dst[0] = *src++; + dst += num_channels; + } + + offset++; + } + + // if the block is stereo, and we don't have room for two more channels, just copy one + // and flag an error + + else if(offset == num_channels - 1) + { + while(samcnt--) + { + dst[0] = src[0]; + dst += num_channels; + src += 2; + } + + wpc->crc_errors++; + offset++; + } + + // otherwise copy the stereo samples into the destination + + else + { + while(samcnt--) + { + dst[0] = *src++; + dst[1] = *src++; + dst += num_channels; + } + + offset += 2; + } + + // check several clues that we're done with this set of blocks and exit if we are; else do next stream + + if((wps->wphdr.flags & FINAL_BLOCK) || wpc->current_stream == wpc->max_streams - 1 || + offset == num_channels) + break; + else + wpc->current_stream++; + } + + // if we didn't get all the channels we expected, mute the buffer and flag an error + + if(offset != num_channels) + { + if(wps->wphdr.flags & DSD_FLAG) + { + int samples_to_zero = samples_to_unpack * num_channels; + int32_t *zptr = bptr; + + while(samples_to_zero--) *zptr++ = 0x55; + } + else + memset(bptr, 0, samples_to_unpack * num_channels * 4); + + wpc->crc_errors++; + } + + // go back to the first stream (we're going to leave them all loaded for now because they might have more + // samples) and free the temp buffer + + wps = wpc->streams[wpc->current_stream = 0]; + free(temp_buffer); + } + // catch the error situation where we have only one channel but run into a stereo block + // (this avoids overwriting the caller's buffer) + else if(!(wps->wphdr.flags & MONO_FLAG) && (num_channels == 1 || wpc->reduced_channels == 1)) + { + memset(bptr, 0, samples_to_unpack * sizeof(*bptr)); + wps->sample_index += samples_to_unpack; + wpc->crc_errors++; + } +#ifdef ENABLE_DSD + else if(wps->wphdr.flags & DSD_FLAG) + unpack_dsd_samples(wpc, bptr, samples_to_unpack); +#endif + else + unpack_samples(wpc, bptr, samples_to_unpack); + + if(file_done) + { + strcpy(wpc->error_message, "can't read all of last block!"); + break; + } + + if(wpc->reduced_channels) + bptr += samples_to_unpack * wpc->reduced_channels; + else + bptr += samples_to_unpack * num_channels; + + samples_unpacked += samples_to_unpack; + samples -= samples_to_unpack; + + // if we just finished a block, check for a calculated crc error + // (and back up the streams a little if possible in case we passed a header) + + if(wps->sample_index == GET_BLOCK_INDEX(wps->wphdr) + wps->wphdr.block_samples) + { + if(check_crc_error(wpc)) + { + int32_t *zptr = bptr, zvalue = (wps->wphdr.flags & DSD_FLAG) ? 0x55 : 0; + uint32_t samples_to_zero = wps->wphdr.block_samples; + + if(samples_to_zero > samples_to_unpack) samples_to_zero = samples_to_unpack; + + samples_to_zero *= (wpc->reduced_channels ? wpc->reduced_channels : num_channels); + + while(samples_to_zero--) *--zptr = zvalue; + + if(wps->blockbuff && wpc->reader->can_seek(wpc->wv_in)) + { + int32_t rseek = ((WavpackHeader *)wps->blockbuff)->ckSize / 3; + wpc->reader->set_pos_rel(wpc->wv_in, (rseek > 16384) ? -16384 : -rseek, SEEK_CUR); + } + + if(wpc->wvc_flag && wps->block2buff && wpc->reader->can_seek(wpc->wvc_in)) + { + int32_t rseek = ((WavpackHeader *)wps->block2buff)->ckSize / 3; + wpc->reader->set_pos_rel(wpc->wvc_in, (rseek > 16384) ? -16384 : -rseek, SEEK_CUR); + } + + wpc->crc_errors++; + } + } + + if(wpc->total_samples != -1 && wps->sample_index == wpc->total_samples) break; + } + +#ifdef ENABLE_DSD + if(wpc->decimation_context) decimate_dsd_run(wpc->decimation_context, buffer, samples_unpacked); +#endif + + return samples_unpacked; +} diff --git a/wavpack/wavpack.h b/wavpack/wavpack.h new file mode 100644 index 0000000..51ca3ed --- /dev/null +++ b/wavpack/wavpack.h @@ -0,0 +1,436 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2016 David Bryant. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// wavpack.h + +#ifndef WAVPACK_H +#define WAVPACK_H + +// This header file contains all the definitions required to use the +// functions in "wputils.c" to read and write WavPack files and streams. + +#include + +#if defined(_MSC_VER) && _MSC_VER < 1600 +typedef unsigned __int64 uint64_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int8 uint8_t; +typedef __int64 int64_t; +typedef __int32 int32_t; +typedef __int16 int16_t; +typedef __int8 int8_t; +#else +#include +#endif + +// RIFF / wav header formats (these occur at the beginning of both wav files +// and pre-4.0 WavPack files that are not in the "raw" mode). Generally, an +// application using the library to read or write WavPack files will not be +// concerned with any of these. + +typedef struct +{ + char ckID[4]; + uint32_t ckSize; + char formType[4]; +} RiffChunkHeader; + +/*typedef struct { + char ckID [4]; + uint32_t ckSize; +} ChunkHeader;*/ + +#define ChunkHeaderFormat "4L" + +typedef struct +{ + uint16_t FormatTag, NumChannels; + uint32_t SampleRate, BytesPerSecond; + uint16_t BlockAlign, BitsPerSample; + uint16_t cbSize, ValidBitsPerSample; + int32_t ChannelMask; + uint16_t SubFormat; + char GUID[14]; +} WaveHeader; + +#define WaveHeaderFormat "SSLLSSSSLS" + +// This is the ONLY structure that occurs in WavPack files (as of version +// 4.0), and is the preamble to every block in both the .wv and .wvc +// files (in little-endian format). Normally, this structure has no use +// to an application using the library to read or write WavPack files, +// but if an application needs to manually parse WavPack files then this +// would be used (with appropriate endian correction). + +typedef struct +{ + char ckID[4]; + uint32_t ckSize; + int16_t version; + unsigned char block_index_u8; + unsigned char total_samples_u8; + uint32_t total_samples, block_index, block_samples, flags, crc; +} WavpackHeader; + +#define WavpackHeaderFormat "4LS2LLLLL" + +// Macros to access the 40-bit block_index field + +#define GET_BLOCK_INDEX(hdr) ((int64_t)(hdr).block_index + ((int64_t)(hdr).block_index_u8 << 32)) + +#define SET_BLOCK_INDEX(hdr, value) \ + do \ + { \ + int64_t tmp = (value); \ + (hdr).block_index = (uint32_t)tmp; \ + (hdr).block_index_u8 = (unsigned char)(tmp >> 32); \ + } while(0) + +// Macros to access the 40-bit total_samples field, which is complicated by the fact that +// all 1's in the lower 32 bits indicates "unknown" (regardless of upper 8 bits) + +#define GET_TOTAL_SAMPLES(hdr) \ + (((hdr).total_samples == (uint32_t)-1) \ + ? -1 \ + : (int64_t)(hdr).total_samples + ((int64_t)(hdr).total_samples_u8 << 32) - (hdr).total_samples_u8) + +#define SET_TOTAL_SAMPLES(hdr, value) \ + do \ + { \ + int64_t tmp = (value); \ + if(tmp < 0) \ + (hdr).total_samples = (uint32_t)-1; \ + else \ + { \ + tmp += (tmp / 0xffffffffLL); \ + (hdr).total_samples = (uint32_t)tmp; \ + (hdr).total_samples_u8 = (unsigned char)(tmp >> 32); \ + } \ + } while(0) + +// or-values for WavpackHeader.flags +#define BYTES_STORED 3 // 1-4 bytes/sample +#define MONO_FLAG 4 // not stereo +#define HYBRID_FLAG 8 // hybrid mode +#define JOINT_STEREO 0x10 // joint stereo +#define CROSS_DECORR 0x20 // no-delay cross decorrelation +#define HYBRID_SHAPE 0x40 // noise shape (hybrid mode only) +#define FLOAT_DATA 0x80 // ieee 32-bit floating point data + +#define INT32_DATA 0x100 // special extended int handling +#define HYBRID_BITRATE 0x200 // bitrate noise (hybrid mode only) +#define HYBRID_BALANCE 0x400 // balance noise (hybrid stereo mode only) + +#define INITIAL_BLOCK 0x800 // initial block of multichannel segment +#define FINAL_BLOCK 0x1000 // final block of multichannel segment + +#define SHIFT_LSB 13 +#define SHIFT_MASK (0x1fL << SHIFT_LSB) + +#define MAG_LSB 18 +#define MAG_MASK (0x1fL << MAG_LSB) + +#define SRATE_LSB 23 +#define SRATE_MASK (0xfL << SRATE_LSB) + +#define FALSE_STEREO 0x40000000 // block is stereo, but data is mono +#define NEW_SHAPING 0x20000000 // use IIR filter for negative shaping + +#define MONO_DATA (MONO_FLAG | FALSE_STEREO) + +// Introduced in WavPack 5.0: +#define HAS_CHECKSUM 0x10000000 // block contains a trailing checksum +#define DSD_FLAG 0x80000000 // block is encoded DSD (1-bit PCM) + +#define IGNORED_FLAGS 0x08000000 // reserved, but ignore if encountered +#define UNKNOWN_FLAGS 0x00000000 // we no longer have any of these spares + +#define MIN_STREAM_VERS 0x402 // lowest stream version we'll decode +#define MAX_STREAM_VERS 0x410 // highest stream version we'll decode or encode + +// These are the mask bit definitions for the metadata chunk id byte (see format.txt) + +#define ID_UNIQUE 0x3f +#define ID_OPTIONAL_DATA 0x20 +#define ID_ODD_SIZE 0x40 +#define ID_LARGE 0x80 + +#define ID_DUMMY 0x0 +#define ID_ENCODER_INFO 0x1 +#define ID_DECORR_TERMS 0x2 +#define ID_DECORR_WEIGHTS 0x3 +#define ID_DECORR_SAMPLES 0x4 +#define ID_ENTROPY_VARS 0x5 +#define ID_HYBRID_PROFILE 0x6 +#define ID_SHAPING_WEIGHTS 0x7 +#define ID_FLOAT_INFO 0x8 +#define ID_INT32_INFO 0x9 +#define ID_WV_BITSTREAM 0xa +#define ID_WVC_BITSTREAM 0xb +#define ID_WVX_BITSTREAM 0xc +#define ID_CHANNEL_INFO 0xd + +#define ID_RIFF_HEADER (ID_OPTIONAL_DATA | 0x1) +#define ID_RIFF_TRAILER (ID_OPTIONAL_DATA | 0x2) +#define ID_ALT_HEADER (ID_OPTIONAL_DATA | 0x3) +#define ID_ALT_TRAILER (ID_OPTIONAL_DATA | 0x4) +#define ID_CONFIG_BLOCK (ID_OPTIONAL_DATA | 0x5) +#define ID_MD5_CHECKSUM (ID_OPTIONAL_DATA | 0x6) +#define ID_SAMPLE_RATE (ID_OPTIONAL_DATA | 0x7) +#define ID_ALT_EXTENSION (ID_OPTIONAL_DATA | 0x8) +#define ID_ALT_MD5_CHECKSUM (ID_OPTIONAL_DATA | 0x9) +#define ID_NEW_CONFIG_BLOCK (ID_OPTIONAL_DATA | 0xa) +#define ID_BLOCK_CHECKSUM (ID_OPTIONAL_DATA | 0xf) + +///////////////////////// WavPack Configuration /////////////////////////////// + +// This external structure is used during encode to provide configuration to +// the encoding engine and during decoding to provide fle information back to +// the higher level functions. Not all fields are used in both modes. + +typedef struct +{ + float bitrate, shaping_weight; + int bits_per_sample, bytes_per_sample; + int qmode, flags, xmode, num_channels, float_norm_exp; + int32_t block_samples, extra_flags, sample_rate, channel_mask; + unsigned char md5_checksum[16], md5_read; + int num_tag_strings; // this field is not used + char **tag_strings; // this field is not used +} WavpackConfig; + +#define CONFIG_HYBRID_FLAG 8 // hybrid mode +#define CONFIG_JOINT_STEREO 0x10 // joint stereo +#define CONFIG_CROSS_DECORR 0x20 // no-delay cross decorrelation +#define CONFIG_HYBRID_SHAPE 0x40 // noise shape (hybrid mode only) +#define CONFIG_FAST_FLAG 0x200 // fast mode +#define CONFIG_HIGH_FLAG 0x800 // high quality mode +#define CONFIG_VERY_HIGH_FLAG 0x1000 // very high +#define CONFIG_BITRATE_KBPS 0x2000 // bitrate is kbps, not bits / sample +#define CONFIG_SHAPE_OVERRIDE 0x8000 // shaping mode specified +#define CONFIG_JOINT_OVERRIDE 0x10000 // joint-stereo mode specified +#define CONFIG_DYNAMIC_SHAPING 0x20000 // dynamic noise shaping +#define CONFIG_CREATE_EXE 0x40000 // create executable +#define CONFIG_CREATE_WVC 0x80000 // create correction file +#define CONFIG_OPTIMIZE_WVC 0x100000 // maximize bybrid compression +#define CONFIG_COMPATIBLE_WRITE 0x400000 // write files for decoders < 4.3 +#define CONFIG_CALC_NOISE 0x800000 // calc noise in hybrid mode +#define CONFIG_EXTRA_MODE 0x2000000 // extra processing mode +#define CONFIG_SKIP_WVX 0x4000000 // no wvx stream w/ floats & big ints +#define CONFIG_MD5_CHECKSUM 0x8000000 // store MD5 signature +#define CONFIG_MERGE_BLOCKS 0x10000000 // merge blocks of equal redundancy (for lossyWAV) +#define CONFIG_PAIR_UNDEF_CHANS 0x20000000 // encode undefined channels in stereo pairs +#define CONFIG_OPTIMIZE_MONO 0x80000000 // optimize for mono streams posing as stereo + +// The lower 8 bits of qmode indicate the use of new features in version 5 that (presently) +// only apply to Core Audio Files (CAF) and DSD files, but could apply to other things too. +// These flags are stored in the file and can be retrieved by a decoder that is aware of +// them, but the individual bits are meaningless to the library. If ANY of these bits are +// set then the MD5 sum is written with a new ID so that old decoders will not see it +// (because these features will cause the MD5 sum to be different and fail). + +#define QMODE_BIG_ENDIAN 0x1 // big-endian data format (opposite of WAV format) +#define QMODE_SIGNED_BYTES 0x2 // 8-bit audio data is signed (opposite of WAV format) +#define QMODE_UNSIGNED_WORDS 0x4 // audio data (other than 8-bit) is unsigned (opposite of WAV format) +#define QMODE_REORDERED_CHANS 0x8 // source channels were not Microsoft order, so they were reordered +#define QMODE_DSD_LSB_FIRST 0x10 // DSD bytes, LSB first (most Sony .dsf files) +#define QMODE_DSD_MSB_FIRST 0x20 // DSD bytes, MSB first (Philips .dff files) +#define QMODE_DSD_IN_BLOCKS 0x40 // DSD data is blocked by channels (Sony .dsf only) +#define QMODE_DSD_AUDIO (QMODE_DSD_LSB_FIRST | QMODE_DSD_MSB_FIRST) + +// The rest of the qmode word is reserved for the private use of the command-line programs +// and are ignored by the library (and not stored either). They really should not be defined +// here, but I thought it would be a good idea to have all the definitions together. + +#define QMODE_ADOBE_MODE 0x100 // user specified Adobe mode +#define QMODE_NO_STORE_WRAPPER 0x200 // user specified to not store audio file wrapper (RIFF, CAFF, etc.) +#define QMODE_CHANS_UNASSIGNED 0x400 // user specified "..." in --channel-order option +#define QMODE_IGNORE_LENGTH 0x800 // user specified to ignore length in file header +#define QMODE_RAW_PCM 0x1000 // user specified raw PCM format (no header present) + +////////////// Callbacks used for reading & writing WavPack streams ////////// + +typedef struct +{ + int32_t (*read_bytes)(void *id, void *data, int32_t bcount); + uint32_t (*get_pos)(void *id); + int (*set_pos_abs)(void *id, uint32_t pos); + int (*set_pos_rel)(void *id, int32_t delta, int mode); + int (*push_back_byte)(void *id, int c); + uint32_t (*get_length)(void *id); + int (*can_seek)(void *id); + + // this callback is for writing edited tags only + int32_t (*write_bytes)(void *id, void *data, int32_t bcount); +} WavpackStreamReader; + +// Extended version of structure for handling large files and added +// functionality for truncating and closing files + +typedef struct +{ + int32_t (*read_bytes)(void *id, void *data, int32_t bcount); + int32_t (*write_bytes)(void *id, void *data, int32_t bcount); + int64_t (*get_pos)(void *id); // new signature for large files + int (*set_pos_abs)(void *id, int64_t pos); // new signature for large files + int (*set_pos_rel)(void *id, int64_t delta, int mode); // new signature for large files + int (*push_back_byte)(void *id, int c); + int64_t (*get_length)(void *id); // new signature for large files + int (*can_seek)(void *id); + int (*truncate_here)(void *id); // new function to truncate file at current position + int (*close)(void *id); // new function to close file +} WavpackStreamReader64; + +typedef int (*WavpackBlockOutput)(void *id, void *data, int32_t bcount); + +//////////////////////////// function prototypes ///////////////////////////// + +typedef void WavpackContext; + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define MAX_WAVPACK_SAMPLES ((1LL << 40) - 257) + + WavpackContext *WavpackOpenRawDecoder(void *main_data, int32_t main_size, void *corr_data, int32_t corr_size, + int16_t version, char *error, int flags, int norm_offset); + + WavpackContext *WavpackOpenFileInputEx64(WavpackStreamReader64 *reader, void *wv_id, void *wvc_id, char *error, + int flags, int norm_offset); + WavpackContext *WavpackOpenFileInputEx(WavpackStreamReader *reader, void *wv_id, void *wvc_id, char *error, + int flags, int norm_offset); + WavpackContext *WavpackOpenFileInput(const char *infilename, char *error, int flags, int norm_offset); + +#define OPEN_WVC 0x1 // open/read "correction" file +#define OPEN_TAGS 0x2 // read ID3v1 / APEv2 tags (seekable file) +#define OPEN_WRAPPER 0x4 // make audio wrapper available (i.e. RIFF) +#define OPEN_2CH_MAX 0x8 // open multichannel as stereo (no downmix) +#define OPEN_NORMALIZE 0x10 // normalize floating point data to +/- 1.0 +#define OPEN_STREAMING \ + 0x20 // "streaming" mode blindly unpacks blocks + // w/o regard to header file position info +#define OPEN_EDIT_TAGS 0x40 // allow editing of tags +#define OPEN_FILE_UTF8 0x80 // assume filenames are UTF-8 encoded, not ANSI (Windows only) + + // new for version 5 + +#define OPEN_DSD_NATIVE \ + 0x100 // open DSD files as bitstreams + // (returned as 8-bit "samples" stored in 32-bit words) +#define OPEN_DSD_AS_PCM 0x200 // open DSD files as 24-bit PCM (decimated 8x) +#define OPEN_ALT_TYPES \ + 0x400 // application is aware of alternate file types & qmode + // (just affects retrieving wrappers & MD5 checksums) +#define OPEN_NO_CHECKSUM 0x800 // don't verify block checksums before decoding + + int WavpackGetMode(WavpackContext *wpc); + +#define MODE_WVC 0x1 +#define MODE_LOSSLESS 0x2 +#define MODE_HYBRID 0x4 +#define MODE_FLOAT 0x8 +#define MODE_VALID_TAG 0x10 +#define MODE_HIGH 0x20 +#define MODE_FAST 0x40 +#define MODE_EXTRA 0x80 // extra mode used, see MODE_XMODE for possible level +#define MODE_APETAG 0x100 +#define MODE_SFX 0x200 +#define MODE_VERY_HIGH 0x400 +#define MODE_MD5 0x800 +#define MODE_XMODE 0x7000 // mask for extra level (1-6, 0=unknown) +#define MODE_DNS 0x8000 + + int WavpackVerifySingleBlock(unsigned char *buffer, int verify_checksum); + int WavpackGetQualifyMode(WavpackContext *wpc); + char *WavpackGetErrorMessage(WavpackContext *wpc); + int WavpackGetVersion(WavpackContext *wpc); + char *WavpackGetFileExtension(WavpackContext *wpc); + unsigned char WavpackGetFileFormat(WavpackContext *wpc); + uint32_t WavpackUnpackSamples(WavpackContext *wpc, int32_t *buffer, uint32_t samples); + uint32_t WavpackGetNumSamples(WavpackContext *wpc); + int64_t WavpackGetNumSamples64(WavpackContext *wpc); + uint32_t WavpackGetNumSamplesInFrame(WavpackContext *wpc); + uint32_t WavpackGetSampleIndex(WavpackContext *wpc); + int64_t WavpackGetSampleIndex64(WavpackContext *wpc); + int WavpackGetNumErrors(WavpackContext *wpc); + int WavpackLossyBlocks(WavpackContext *wpc); + int WavpackSeekSample(WavpackContext *wpc, uint32_t sample); + int WavpackSeekSample64(WavpackContext *wpc, int64_t sample); + WavpackContext *WavpackCloseFile(WavpackContext *wpc); + uint32_t WavpackGetSampleRate(WavpackContext *wpc); + uint32_t WavpackGetNativeSampleRate(WavpackContext *wpc); + int WavpackGetBitsPerSample(WavpackContext *wpc); + int WavpackGetBytesPerSample(WavpackContext *wpc); + int WavpackGetNumChannels(WavpackContext *wpc); + int WavpackGetChannelMask(WavpackContext *wpc); + int WavpackGetReducedChannels(WavpackContext *wpc); + int WavpackGetFloatNormExp(WavpackContext *wpc); + int WavpackGetMD5Sum(WavpackContext *wpc, unsigned char data[16]); + void WavpackGetChannelIdentities(WavpackContext *wpc, unsigned char *identities); + uint32_t WavpackGetChannelLayout(WavpackContext *wpc, unsigned char *reorder); + uint32_t WavpackGetWrapperBytes(WavpackContext *wpc); + unsigned char *WavpackGetWrapperData(WavpackContext *wpc); + void WavpackFreeWrapper(WavpackContext *wpc); + void WavpackSeekTrailingWrapper(WavpackContext *wpc); + double WavpackGetProgress(WavpackContext *wpc); + uint32_t WavpackGetFileSize(WavpackContext *wpc); + int64_t WavpackGetFileSize64(WavpackContext *wpc); + double WavpackGetRatio(WavpackContext *wpc); + double WavpackGetAverageBitrate(WavpackContext *wpc, int count_wvc); + double WavpackGetInstantBitrate(WavpackContext *wpc); + int WavpackGetNumTagItems(WavpackContext *wpc); + int WavpackGetTagItem(WavpackContext *wpc, const char *item, char *value, int size); + int WavpackGetTagItemIndexed(WavpackContext *wpc, int index, char *item, int size); + int WavpackGetNumBinaryTagItems(WavpackContext *wpc); + int WavpackGetBinaryTagItem(WavpackContext *wpc, const char *item, char *value, int size); + int WavpackGetBinaryTagItemIndexed(WavpackContext *wpc, int index, char *item, int size); + int WavpackAppendTagItem(WavpackContext *wpc, const char *item, const char *value, int vsize); + int WavpackAppendBinaryTagItem(WavpackContext *wpc, const char *item, const char *value, int vsize); + int WavpackDeleteTagItem(WavpackContext *wpc, const char *item); + int WavpackWriteTag(WavpackContext *wpc); + + WavpackContext *WavpackOpenFileOutput(WavpackBlockOutput blockout, void *wv_id, void *wvc_id); + void WavpackSetFileInformation(WavpackContext *wpc, char *file_extension, unsigned char file_format); + +#define WP_FORMAT_WAV 0 // Microsoft RIFF, including BWF and RF64 varients +#define WP_FORMAT_W64 1 // Sony Wave64 +#define WP_FORMAT_CAF 2 // Apple CoreAudio +#define WP_FORMAT_DFF 3 // Philips DSDIFF +#define WP_FORMAT_DSF 4 // Sony DSD Format + + int WavpackSetConfiguration(WavpackContext *wpc, WavpackConfig *config, uint32_t total_samples); + int WavpackSetConfiguration64(WavpackContext *wpc, WavpackConfig *config, int64_t total_samples, + const unsigned char *chan_ids); + int WavpackSetChannelLayout(WavpackContext *wpc, uint32_t layout_tag, const unsigned char *reorder); + int WavpackAddWrapper(WavpackContext *wpc, void *data, uint32_t bcount); + int WavpackStoreMD5Sum(WavpackContext *wpc, unsigned char data[16]); + int WavpackPackInit(WavpackContext *wpc); + int WavpackPackSamples(WavpackContext *wpc, int32_t *sample_buffer, uint32_t sample_count); + int WavpackFlushSamples(WavpackContext *wpc); + void WavpackUpdateNumSamples(WavpackContext *wpc, void *first_block); + void *WavpackGetWrapperLocation(void *first_block, uint32_t *size); + double WavpackGetEncodedNoise(WavpackContext *wpc, double *peak); + + void WavpackFloatNormalize(int32_t *values, int32_t num_values, int delta_exp); + + void WavpackLittleEndianToNative(void *data, char *format); + void WavpackNativeToLittleEndian(void *data, char *format); + void WavpackBigEndianToNative(void *data, char *format); + void WavpackNativeToBigEndian(void *data, char *format); + + uint32_t WavpackGetLibraryVersion(void); + const char *WavpackGetLibraryVersionString(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/wavpack/wavpack_local.h b/wavpack/wavpack_local.h new file mode 100644 index 0000000..4605627 --- /dev/null +++ b/wavpack/wavpack_local.h @@ -0,0 +1,945 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2013 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// wavpack_local.h + +#ifndef WAVPACK_LOCAL_H +#define WAVPACK_LOCAL_H + +#if defined(_WIN32) +#define strdup(x) _strdup(x) +#define FASTCALL __fastcall +#else +#define FASTCALL +#endif + +#if defined(_WIN32) || (defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && (BYTE_ORDER == LITTLE_ENDIAN)) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) +#define BITSTREAM_SHORTS // use 16-bit "shorts" for reading/writing bitstreams (instead of chars) +// (only works on little-endian machines) +#endif + +#include + +// This header file contains all the definitions required by WavPack. + +#if defined(_MSC_VER) && _MSC_VER < 1600 +#include +typedef unsigned __int64 uint64_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int8 uint8_t; +typedef __int64 int64_t; +typedef __int32 int32_t; +typedef __int16 int16_t; +typedef __int8 int8_t; +#else +#include +#endif + +// Because the C99 specification states that "The order of allocation of +// bit-fields within a unit (high-order to low-order or low-order to +// high-order) is implementation-defined" (6.7.2.1), I decided to change +// the representation of floating-point values from a structure of +// bit-fields to a 32-bit integer with access macros. Note that the WavPack +// library doesn't use any floating-point math to implement compression of +// floating-point data (although a little floating-point math is used in +// high-level functions unrelated to the codec). + +typedef int32_t f32; + +#define get_mantissa(f) ((f) & 0x7fffff) +#define get_magnitude(f) ((f) & 0x7fffffff) +#define get_exponent(f) (((f) >> 23) & 0xff) +#define get_sign(f) (((f) >> 31) & 0x1) + +#define set_mantissa(f, v) (f) ^= (((f) ^ (v)) & 0x7fffff) +#define set_exponent(f, v) (f) ^= (((f) ^ ((v) << 23)) & 0x7f800000) +#define set_sign(f, v) (f) ^= (((f) ^ ((v) << 31)) & 0x80000000) + +#include + +#define FALSE 0 +#define TRUE 1 + +// ID3v1 and APEv2 TAG formats (may occur at the end of WavPack files) + +typedef struct +{ + char tag_id[3], title[30], artist[30], album[30]; + char year[4], comment[30], genre; +} ID3_Tag; + +typedef struct +{ + char ID[8]; + int32_t version, length, item_count, flags; + char res[8]; +} APE_Tag_Hdr; + +#define APE_Tag_Hdr_Format "8LLLL" + +#define APE_TAG_TYPE_TEXT 0x0 +#define APE_TAG_TYPE_BINARY 0x1 +#define APE_TAG_THIS_IS_HEADER 0x20000000 +#define APE_TAG_CONTAINS_HEADER 0x80000000 +#define APE_TAG_MAX_LENGTH (1024 * 1024 * 16) + +typedef struct +{ + int64_t tag_file_pos; + int tag_begins_file; + ID3_Tag id3_tag; + APE_Tag_Hdr ape_tag_hdr; + unsigned char *ape_tag_data; +} M_Tag; + +// RIFF / wav header formats (these occur at the beginning of both wav files +// and pre-4.0 WavPack files that are not in the "raw" mode) + +typedef struct +{ + char ckID[4]; + uint32_t ckSize; + char formType[4]; +} RiffChunkHeader; + +typedef struct +{ + char ckID[4]; + uint32_t ckSize; +} ChunkHeader; + +#define ChunkHeaderFormat "4L" + +typedef struct +{ + uint16_t FormatTag, NumChannels; + uint32_t SampleRate, BytesPerSecond; + uint16_t BlockAlign, BitsPerSample; + uint16_t cbSize, ValidBitsPerSample; + int32_t ChannelMask; + uint16_t SubFormat; + char GUID[14]; +} WaveHeader; + +#define WaveHeaderFormat "SSLLSSSSLS" + +////////////////////////////// WavPack Header ///////////////////////////////// + +// Note that this is the ONLY structure that is written to (or read from) +// WavPack 4.0 files, and is the preamble to every block in both the .wv +// and .wvc files. + +typedef struct +{ + char ckID[4]; + uint32_t ckSize; + int16_t version; + unsigned char block_index_u8; + unsigned char total_samples_u8; + uint32_t total_samples, block_index, block_samples, flags, crc; +} WavpackHeader; + +#define WavpackHeaderFormat "4LS2LLLLL" + +// Macros to access the 40-bit block_index field + +#define GET_BLOCK_INDEX(hdr) ((int64_t)(hdr).block_index + ((int64_t)(hdr).block_index_u8 << 32)) + +#define SET_BLOCK_INDEX(hdr, value) \ + do \ + { \ + int64_t tmp = (value); \ + (hdr).block_index = (uint32_t)tmp; \ + (hdr).block_index_u8 = (unsigned char)(tmp >> 32); \ + } while(0) + +// Macros to access the 40-bit total_samples field, which is complicated by the fact that +// all 1's in the lower 32 bits indicates "unknown" (regardless of upper 8 bits) + +#define GET_TOTAL_SAMPLES(hdr) \ + (((hdr).total_samples == (uint32_t)-1) \ + ? -1 \ + : (int64_t)(hdr).total_samples + ((int64_t)(hdr).total_samples_u8 << 32) - (hdr).total_samples_u8) + +#define SET_TOTAL_SAMPLES(hdr, value) \ + do \ + { \ + int64_t tmp = (value); \ + if(tmp < 0) \ + (hdr).total_samples = (uint32_t)-1; \ + else \ + { \ + tmp += (tmp / (int64_t)0xffffffff); \ + (hdr).total_samples = (uint32_t)tmp; \ + (hdr).total_samples_u8 = (unsigned char)(tmp >> 32); \ + } \ + } while(0) + +// or-values for "flags" + +#define BYTES_STORED 3 // 1-4 bytes/sample +#define MONO_FLAG 4 // not stereo +#define HYBRID_FLAG 8 // hybrid mode +#define JOINT_STEREO 0x10 // joint stereo +#define CROSS_DECORR 0x20 // no-delay cross decorrelation +#define HYBRID_SHAPE 0x40 // noise shape (hybrid mode only) +#define FLOAT_DATA 0x80 // ieee 32-bit floating point data + +#define INT32_DATA 0x100 // special extended int handling +#define HYBRID_BITRATE 0x200 // bitrate noise (hybrid mode only) +#define HYBRID_BALANCE 0x400 // balance noise (hybrid stereo mode only) + +#define INITIAL_BLOCK 0x800 // initial block of multichannel segment +#define FINAL_BLOCK 0x1000 // final block of multichannel segment + +#define SHIFT_LSB 13 +#define SHIFT_MASK (0x1fL << SHIFT_LSB) + +#define MAG_LSB 18 +#define MAG_MASK (0x1fL << MAG_LSB) + +#define SRATE_LSB 23 +#define SRATE_MASK (0xfL << SRATE_LSB) + +#define FALSE_STEREO 0x40000000 // block is stereo, but data is mono +#define NEW_SHAPING 0x20000000 // use IIR filter for negative shaping + +#define MONO_DATA (MONO_FLAG | FALSE_STEREO) + +// Introduced in WavPack 5.0: +#define HAS_CHECKSUM 0x10000000 // block contains a trailing checksum +#define DSD_FLAG 0x80000000 // block is encoded DSD (1-bit PCM) + +#define IGNORED_FLAGS 0x08000000 // reserved, but ignore if encountered +#define UNKNOWN_FLAGS 0x00000000 // we no longer have any of these spares + +#define MIN_STREAM_VERS 0x402 // lowest stream version we'll decode +#define MAX_STREAM_VERS 0x410 // highest stream version we'll decode or encode +// (only stream version to support mono optimization) +#define CUR_STREAM_VERS 0x407 // universally compatible stream version + +//////////////////////////// WavPack Metadata ///////////////////////////////// + +// This is an internal representation of metadata. + +typedef struct +{ + int32_t byte_length; + void *data; + unsigned char id; +} WavpackMetadata; + +#define ID_UNIQUE 0x3f +#define ID_OPTIONAL_DATA 0x20 +#define ID_ODD_SIZE 0x40 +#define ID_LARGE 0x80 + +#define ID_DUMMY 0x0 +#define ID_ENCODER_INFO 0x1 +#define ID_DECORR_TERMS 0x2 +#define ID_DECORR_WEIGHTS 0x3 +#define ID_DECORR_SAMPLES 0x4 +#define ID_ENTROPY_VARS 0x5 +#define ID_HYBRID_PROFILE 0x6 +#define ID_SHAPING_WEIGHTS 0x7 +#define ID_FLOAT_INFO 0x8 +#define ID_INT32_INFO 0x9 +#define ID_WV_BITSTREAM 0xa +#define ID_WVC_BITSTREAM 0xb +#define ID_WVX_BITSTREAM 0xc +#define ID_CHANNEL_INFO 0xd +#define ID_DSD_BLOCK 0xe + +#define ID_RIFF_HEADER (ID_OPTIONAL_DATA | 0x1) +#define ID_RIFF_TRAILER (ID_OPTIONAL_DATA | 0x2) +#define ID_ALT_HEADER (ID_OPTIONAL_DATA | 0x3) +#define ID_ALT_TRAILER (ID_OPTIONAL_DATA | 0x4) +#define ID_CONFIG_BLOCK (ID_OPTIONAL_DATA | 0x5) +#define ID_MD5_CHECKSUM (ID_OPTIONAL_DATA | 0x6) +#define ID_SAMPLE_RATE (ID_OPTIONAL_DATA | 0x7) +#define ID_ALT_EXTENSION (ID_OPTIONAL_DATA | 0x8) +#define ID_ALT_MD5_CHECKSUM (ID_OPTIONAL_DATA | 0x9) +#define ID_NEW_CONFIG_BLOCK (ID_OPTIONAL_DATA | 0xa) +#define ID_CHANNEL_IDENTITIES (ID_OPTIONAL_DATA | 0xb) +#define ID_BLOCK_CHECKSUM (ID_OPTIONAL_DATA | 0xf) + +///////////////////////// WavPack Configuration /////////////////////////////// + +// This internal structure is used during encode to provide configuration to +// the encoding engine and during decoding to provide fle information back to +// the higher level functions. Not all fields are used in both modes. + +typedef struct +{ + float bitrate, shaping_weight; + int bits_per_sample, bytes_per_sample; + int qmode, flags, xmode, num_channels, float_norm_exp; + int32_t block_samples, extra_flags, sample_rate, channel_mask; + unsigned char md5_checksum[16], md5_read; + int num_tag_strings; + char **tag_strings; +} WavpackConfig; + +#define CONFIG_BYTES_STORED 3 // 1-4 bytes/sample +#define CONFIG_MONO_FLAG 4 // not stereo +#define CONFIG_HYBRID_FLAG 8 // hybrid mode +#define CONFIG_JOINT_STEREO 0x10 // joint stereo +#define CONFIG_CROSS_DECORR 0x20 // no-delay cross decorrelation +#define CONFIG_HYBRID_SHAPE 0x40 // noise shape (hybrid mode only) +#define CONFIG_FLOAT_DATA 0x80 // ieee 32-bit floating point data + +#define CONFIG_FAST_FLAG 0x200 // fast mode +#define CONFIG_HIGH_FLAG 0x800 // high quality mode +#define CONFIG_VERY_HIGH_FLAG 0x1000 // very high +#define CONFIG_BITRATE_KBPS 0x2000 // bitrate is kbps, not bits / sample +#define CONFIG_AUTO_SHAPING 0x4000 // automatic noise shaping +#define CONFIG_SHAPE_OVERRIDE 0x8000 // shaping mode specified +#define CONFIG_JOINT_OVERRIDE 0x10000 // joint-stereo mode specified +#define CONFIG_DYNAMIC_SHAPING 0x20000 // dynamic noise shaping +#define CONFIG_CREATE_EXE 0x40000 // create executable +#define CONFIG_CREATE_WVC 0x80000 // create correction file +#define CONFIG_OPTIMIZE_WVC 0x100000 // maximize bybrid compression +#define CONFIG_COMPATIBLE_WRITE 0x400000 // write files for decoders < 4.3 +#define CONFIG_CALC_NOISE 0x800000 // calc noise in hybrid mode +#define CONFIG_LOSSY_MODE 0x1000000 // obsolete (for information) +#define CONFIG_EXTRA_MODE 0x2000000 // extra processing mode +#define CONFIG_SKIP_WVX 0x4000000 // no wvx stream w/ floats & big ints +#define CONFIG_MD5_CHECKSUM 0x8000000 // compute & store MD5 signature +#define CONFIG_MERGE_BLOCKS 0x10000000 // merge blocks of equal redundancy (for lossyWAV) +#define CONFIG_PAIR_UNDEF_CHANS 0x20000000 // encode undefined channels in stereo pairs +#define CONFIG_OPTIMIZE_MONO 0x80000000 // optimize for mono streams posing as stereo + +#define QMODE_DSD_AUDIO 0x30 // if either of these is set in qmode (version 5.0) + +/* + * These config flags were never actually used, or are no longer used, or are + * used for something else now. They may be used in the future for what they + * say, or for something else. WavPack files in the wild *may* have some of + * these bit set in their config flags (with these older meanings), but only + * if the stream version is 0x410 or less than 0x407. Of course, this is not + * very important because once the file has been encoded, the config bits are + * just for information purposes (i.e., they do not affect decoding), + * + #define CONFIG_ADOBE_MODE 0x100 // "adobe" mode for 32-bit floats + #define CONFIG_VERY_FAST_FLAG 0x400 // double fast + #define CONFIG_COPY_TIME 0x20000 // copy file-time from source + #define CONFIG_QUALITY_MODE 0x200000 // psychoacoustic quality mode + #define CONFIG_RAW_FLAG 0x400000 // raw mode (not implemented yet) + #define CONFIG_QUIET_MODE 0x10000000 // don't report progress % + #define CONFIG_IGNORE_LENGTH 0x20000000 // ignore length in wav header + #define CONFIG_NEW_RIFF_HEADER 0x40000000 // generate new RIFF wav header + * + */ + +#define EXTRA_SCAN_ONLY 1 +#define EXTRA_STEREO_MODES 2 +#define EXTRA_TRY_DELTAS 8 +#define EXTRA_ADJUST_DELTAS 16 +#define EXTRA_SORT_FIRST 32 +#define EXTRA_BRANCHES 0x1c0 +#define EXTRA_SKIP_8TO16 512 +#define EXTRA_TERMS 0x3c00 +#define EXTRA_DUMP_TERMS 16384 +#define EXTRA_SORT_LAST 32768 + +//////////////////////////////// WavPack Stream /////////////////////////////// + +// This internal structure contains everything required to handle a WavPack +// "stream", which is defined as a stereo or mono stream of audio samples. For +// multichannel audio several of these would be required. Each stream contains +// pointers to hold a complete allocated block of WavPack data, although it's +// possible to decode WavPack blocks without buffering an entire block. + +typedef struct bs +{ +#ifdef BITSTREAM_SHORTS + uint16_t *buf, *end, *ptr; +#else + unsigned char *buf, *end, *ptr; +#endif + void (*wrap)(struct bs *bs); + int error, bc; + uint32_t sr; +} Bitstream; + +#define MAX_WRAPPER_BYTES 16777216 +#define NEW_MAX_STREAMS 4096 +#define OLD_MAX_STREAMS 8 +#define MAX_NTERMS 16 +#define MAX_TERM 8 + +// Note that this structure is directly accessed in assembly files, so modify with care + +struct decorr_pass +{ + int32_t term, delta, weight_A, weight_B; + int32_t samples_A[MAX_TERM], samples_B[MAX_TERM]; + int32_t aweight_A, aweight_B; + int32_t sum_A, sum_B; +}; + +typedef struct +{ + signed char joint_stereo, delta, terms[MAX_NTERMS + 1]; +} WavpackDecorrSpec; + +struct entropy_data +{ + uint32_t median[3], slow_level, error_limit; +}; + +struct words_data +{ + uint32_t bitrate_delta[2], bitrate_acc[2]; + uint32_t pend_data, holding_one, zeros_acc; + int holding_zero, pend_count; + struct entropy_data c[2]; +}; + +typedef struct +{ + int32_t value, filter0, filter1, filter2, filter3, filter4, filter5, filter6, factor, byte; +} DSDfilters; + +typedef struct +{ + WavpackHeader wphdr; + struct words_data w; + + unsigned char *blockbuff, *blockend; + unsigned char *block2buff, *block2end; + int32_t *sample_buffer; + + int64_t sample_index; + int bits, num_terms, mute_error, joint_stereo, false_stereo, shift; + int num_decorrs, num_passes, best_decorr, mask_decorr; + uint32_t crc, crc_x, crc_wvx; + Bitstream wvbits, wvcbits, wvxbits; + int init_done, wvc_skip; + float delta_decay; + + unsigned char int32_sent_bits, int32_zeros, int32_ones, int32_dups; + unsigned char float_flags, float_shift, float_max_exp, float_norm_exp; + + struct + { + int32_t shaping_acc[2], shaping_delta[2], error[2]; + double noise_sum, noise_ave, noise_max; + int16_t *shaping_data, *shaping_array; + int32_t shaping_samples; + } dc; + + struct decorr_pass decorr_passes[MAX_NTERMS], analysis_pass; + const WavpackDecorrSpec *decorr_specs; + + struct + { + unsigned char *byteptr, *endptr, (*probabilities)[256], **value_lookup, mode, ready; + int history_bins, p0, p1; + int16_t (*summed_probabilities)[256]; + uint32_t low, high, value; + DSDfilters filters[2]; + int32_t *ptable; + } dsd; + +} WavpackStream; + +// flags for float_flags: + +#define FLOAT_SHIFT_ONES 1 // bits left-shifted into float = '1' +#define FLOAT_SHIFT_SAME 2 // bits left-shifted into float are the same +#define FLOAT_SHIFT_SENT 4 // bits shifted into float are sent literally +#define FLOAT_ZEROS_SENT 8 // "zeros" are not all real zeros +#define FLOAT_NEG_ZEROS 0x10 // contains negative zeros +#define FLOAT_EXCEPTIONS 0x20 // contains exceptions (inf, nan, etc.) + +/////////////////////////////// WavPack Context /////////////////////////////// + +// This internal structure holds everything required to encode or decode WavPack +// files. It is recommended that direct access to this structure be minimized +// and the provided utilities used instead. + +typedef struct +{ + int32_t (*read_bytes)(void *id, void *data, int32_t bcount); + uint32_t (*get_pos)(void *id); + int (*set_pos_abs)(void *id, uint32_t pos); + int (*set_pos_rel)(void *id, int32_t delta, int mode); + int (*push_back_byte)(void *id, int c); + uint32_t (*get_length)(void *id); + int (*can_seek)(void *id); + + // this callback is for writing edited tags only + int32_t (*write_bytes)(void *id, void *data, int32_t bcount); +} WavpackStreamReader; + +// Extended version of structure for handling large files and added +// functionality for truncating and closing files + +typedef struct +{ + int32_t (*read_bytes)(void *id, void *data, int32_t bcount); + int32_t (*write_bytes)(void *id, void *data, int32_t bcount); + int64_t (*get_pos)(void *id); // new signature for large files + int (*set_pos_abs)(void *id, int64_t pos); // new signature for large files + int (*set_pos_rel)(void *id, int64_t delta, int mode); // new signature for large files + int (*push_back_byte)(void *id, int c); + int64_t (*get_length)(void *id); // new signature for large files + int (*can_seek)(void *id); + int (*truncate_here)(void *id); // new function to truncate file at current position + int (*close)(void *id); // new function to close file +} WavpackStreamReader64; + +typedef int (*WavpackBlockOutput)(void *id, void *data, int32_t bcount); + +typedef struct +{ + WavpackConfig config; + + WavpackMetadata *metadata; + uint32_t metabytes; + int metacount; + + unsigned char *wrapper_data; + uint32_t wrapper_bytes; + + WavpackBlockOutput blockout; + void *wv_out, *wvc_out; + + WavpackStreamReader64 *reader; + void *wv_in, *wvc_in; + + int64_t filelen, file2len, filepos, file2pos, total_samples, initial_index; + uint32_t crc_errors, first_flags; + int wvc_flag, open_flags, norm_offset, reduced_channels, lossy_blocks, version_five; + uint32_t block_samples, ave_block_samples, block_boundary, max_samples, acc_samples, riff_trailer_bytes; + int riff_header_added, riff_header_created; + M_Tag m_tag; + + int current_stream, num_streams, max_streams, stream_version; + WavpackStream **streams; + void *stream3; + + // these items were added in 5.0 to support alternate file types (especially CAF & DSD) + unsigned char file_format, *channel_reordering, *channel_identities; + uint32_t channel_layout, dsd_multiplier; + void *decimation_context; + char file_extension[8]; + + char error_message[80]; +} WavpackContext; + +//////////////////////// function prototypes and macros ////////////////////// + +#define CLEAR(destin) memset(&destin, 0, sizeof(destin)); + +//////////////////////////////// decorrelation ////////////////////////////// +// modules: pack.c, unpack.c, unpack_floats.c, extra1.c, extra2.c + +// #define SKIP_DECORRELATION // experimental switch to disable all decorrelation on encode + +// These macros implement the weight application and update operations +// that are at the heart of the decorrelation loops. Note that there are +// sometimes two and even three versions of each macro. Theses should be +// equivalent and produce identical results, but some may perform better +// or worse on a given architecture. + +#if 1 // PERFCOND - apply decorrelation weight when no 32-bit overflow possible +#define apply_weight_i(weight, sample) ((weight * sample + 512) >> 10) +#else +#define apply_weight_i(weight, sample) ((((weight * sample) >> 8) + 2) >> 2) +#endif + +#if 1 // PERFCOND - apply decorrelation weight when 32-bit overflow is possible +#define apply_weight_f(weight, sample) \ + (((((sample & 0xffff) * weight) >> 9) + (((sample & ~0xffff) >> 9) * weight) + 1) >> 1) +#elif 1 +#define apply_weight_f(weight, sample) ((int32_t)((weight * (int64_t)sample + 512) >> 10)) +#else +#define apply_weight_f(weight, sample) ((int32_t)floor(((double)weight * sample + 512.0) / 1024.0)) +#endif + +#if 1 // PERFCOND - universal version that checks input magnitude or always uses long version +#define apply_weight(weight, sample) \ + (sample != (int16_t)sample ? apply_weight_f(weight, sample) : apply_weight_i(weight, sample)) +#else +#define apply_weight(weight, sample) (apply_weight_f(weight, sample)) +#endif + +#if 1 // PERFCOND +#define update_weight(weight, delta, source, result) \ + if(source && result) \ + { \ + int32_t s = (int32_t)(source ^ result) >> 31; \ + weight = (delta ^ s) + (weight - s); \ + } +#elif 1 +#define update_weight(weight, delta, source, result) \ + if(source && result) weight += (((source ^ result) >> 30) | 1) * delta; +#else +#define update_weight(weight, delta, source, result) \ + if(source && result) (source ^ result) < 0 ? (weight -= delta) : (weight += delta); +#endif + +#define update_weight_clip(weight, delta, source, result) \ + if(source && result) \ + { \ + const int32_t s = (source ^ result) >> 31; \ + if((weight = (weight ^ s) + (delta - s)) > 1024) weight = 1024; \ + weight = (weight ^ s) - s; \ + } + +void pack_init(WavpackContext *wpc); +int pack_block(WavpackContext *wpc, int32_t *buffer); +void send_general_metadata(WavpackContext *wpc); +void free_metadata(WavpackMetadata *wpmd); +int copy_metadata(WavpackMetadata *wpmd, unsigned char *buffer_start, unsigned char *buffer_end); +double WavpackGetEncodedNoise(WavpackContext *wpc, double *peak); +int unpack_init(WavpackContext *wpc); +int read_decorr_terms(WavpackStream *wps, WavpackMetadata *wpmd); +int read_decorr_weights(WavpackStream *wps, WavpackMetadata *wpmd); +int read_decorr_samples(WavpackStream *wps, WavpackMetadata *wpmd); +int read_shaping_info(WavpackStream *wps, WavpackMetadata *wpmd); +int32_t unpack_samples(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count); +int check_crc_error(WavpackContext *wpc); +int scan_float_data(WavpackStream *wps, f32 *values, int32_t num_values); +void send_float_data(WavpackStream *wps, f32 *values, int32_t num_values); +void float_values(WavpackStream *wps, int32_t *values, int32_t num_values); +void dynamic_noise_shaping(WavpackContext *wpc, int32_t *buffer, int shortening_allowed); +void execute_stereo(WavpackContext *wpc, int32_t *samples, int no_history, int do_samples); +void execute_mono(WavpackContext *wpc, int32_t *samples, int no_history, int do_samples); + +////////////////////////// DSD related (including decimation) ////////////////////////// +// modules: pack_dsd.c unpack_dsd.c + +void pack_dsd_init(WavpackContext *wpc); +int pack_dsd_block(WavpackContext *wpc, int32_t *buffer); +int init_dsd_block(WavpackContext *wpc, WavpackMetadata *wpmd); +int32_t unpack_dsd_samples(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count); + +void *decimate_dsd_init(int num_channels); +void decimate_dsd_reset(void *decimate_context); +void decimate_dsd_run(void *decimate_context, int32_t *samples, int num_samples); +void decimate_dsd_destroy(void *decimate_context); + +///////////////////////////////// CPU feature detection //////////////////////////////// + +int unpack_cpu_has_feature_x86(int findex), pack_cpu_has_feature_x86(int findex); + +#define CPU_FEATURE_MMX 23 + +///////////////////////////// pre-4.0 version decoding //////////////////////////// +// modules: unpack3.c, unpack3_open.c, unpack3_seek.c + +WavpackContext *open_file3(WavpackContext *wpc, char *error); +int32_t unpack_samples3(WavpackContext *wpc, int32_t *buffer, uint32_t sample_count); +int seek_sample3(WavpackContext *wpc, uint32_t desired_index); +uint32_t get_sample_index3(WavpackContext *wpc); +void free_stream3(WavpackContext *wpc); +int get_version3(WavpackContext *wpc); + +////////////////////////////// bitstream macros & functions ///////////////////////////// + +#define bs_is_open(bs) ((bs)->ptr != NULL) +uint32_t bs_close_read(Bitstream *bs); + +#define getbit(bs) \ + ((((bs)->bc) ? ((bs)->bc--, (bs)->sr & 1) \ + : (((++((bs)->ptr) != (bs)->end) ? (void)0 : (bs)->wrap(bs)), \ + (bs)->bc = sizeof(*((bs)->ptr)) * 8 - 1, ((bs)->sr = *((bs)->ptr)) & 1)) \ + ? ((bs)->sr >>= 1, 1) \ + : ((bs)->sr >>= 1, 0)) + +#define getbits(value, nbits, bs) \ + do \ + { \ + while((nbits) > (bs)->bc) \ + { \ + if(++((bs)->ptr) == (bs)->end) (bs)->wrap(bs); \ + (bs)->sr |= (int32_t)*((bs)->ptr) << (bs)->bc; \ + (bs)->bc += sizeof(*((bs)->ptr)) * 8; \ + } \ + *(value) = (bs)->sr; \ + if((bs)->bc > 32) \ + { \ + (bs)->bc -= (nbits); \ + (bs)->sr = *((bs)->ptr) >> (sizeof(*((bs)->ptr)) * 8 - (bs)->bc); \ + } \ + else \ + { \ + (bs)->bc -= (nbits); \ + (bs)->sr >>= (nbits); \ + } \ + } while(0) + +#define putbit(bit, bs) \ + do \ + { \ + if(bit) (bs)->sr |= (1 << (bs)->bc); \ + if(++((bs)->bc) == sizeof(*((bs)->ptr)) * 8) \ + { \ + *((bs)->ptr) = (bs)->sr; \ + (bs)->sr = (bs)->bc = 0; \ + if(++((bs)->ptr) == (bs)->end) (bs)->wrap(bs); \ + } \ + } while(0) + +#define putbit_0(bs) \ + do \ + { \ + if(++((bs)->bc) == sizeof(*((bs)->ptr)) * 8) \ + { \ + *((bs)->ptr) = (bs)->sr; \ + (bs)->sr = (bs)->bc = 0; \ + if(++((bs)->ptr) == (bs)->end) (bs)->wrap(bs); \ + } \ + } while(0) + +#define putbit_1(bs) \ + do \ + { \ + (bs)->sr |= (1 << (bs)->bc); \ + if(++((bs)->bc) == sizeof(*((bs)->ptr)) * 8) \ + { \ + *((bs)->ptr) = (bs)->sr; \ + (bs)->sr = (bs)->bc = 0; \ + if(++((bs)->ptr) == (bs)->end) (bs)->wrap(bs); \ + } \ + } while(0) + +#define putbits(value, nbits, bs) \ + do \ + { \ + (bs)->sr |= (int32_t)(value) << (bs)->bc; \ + if(((bs)->bc += (nbits)) >= sizeof(*((bs)->ptr)) * 8) do \ + { \ + *((bs)->ptr) = (bs)->sr; \ + (bs)->sr >>= sizeof(*((bs)->ptr)) * 8; \ + if(((bs)->bc -= sizeof(*((bs)->ptr)) * 8) > 32 - sizeof(*((bs)->ptr)) * 8) \ + (bs)->sr |= ((value) >> ((nbits) - (bs)->bc)); \ + if(++((bs)->ptr) == (bs)->end) (bs)->wrap(bs); \ + } while((bs)->bc >= sizeof(*((bs)->ptr)) * 8); \ + } while(0) + +///////////////////////////// entropy encoder / decoder //////////////////////////// +// modules: entropy_utils.c, read_words.c, write_words.c + +// these control the time constant "slow_level" which is used for hybrid mode +// that controls bitrate as a function of residual level (HYBRID_BITRATE). +#define SLS 8 +#define SLO ((1 << (SLS - 1))) + +#define LIMIT_ONES 16 // maximum consecutive 1s sent for "div" data + +// these control the time constant of the 3 median level breakpoints +#define DIV0 128 // 5/7 of samples +#define DIV1 64 // 10/49 of samples +#define DIV2 32 // 20/343 of samples + +// this macro retrieves the specified median breakpoint (without frac; min = 1) +#define GET_MED(med) (((c->median[med]) >> 4) + 1) + +// These macros update the specified median breakpoints. Note that the median +// is incremented when the sample is higher than the median, else decremented. +// They are designed so that the median will never drop below 1 and the value +// is essentially stationary if there are 2 increments for every 5 decrements. + +#define INC_MED0() (c->median[0] += ((c->median[0] + DIV0) / DIV0) * 5) +#define DEC_MED0() (c->median[0] -= ((c->median[0] + (DIV0 - 2)) / DIV0) * 2) +#define INC_MED1() (c->median[1] += ((c->median[1] + DIV1) / DIV1) * 5) +#define DEC_MED1() (c->median[1] -= ((c->median[1] + (DIV1 - 2)) / DIV1) * 2) +#define INC_MED2() (c->median[2] += ((c->median[2] + DIV2) / DIV2) * 5) +#define DEC_MED2() (c->median[2] -= ((c->median[2] + (DIV2 - 2)) / DIV2) * 2) + +#ifdef HAVE___BUILTIN_CLZ +#define count_bits(av) ((av) ? 32 - __builtin_clz(av) : 0) +#elif defined(_WIN64) +static __inline int count_bits(uint32_t av) +{ + unsigned long res; + return _BitScanReverse(&res, av) ? (int)(res + 1) : 0; +} +#else +#define count_bits(av) \ + ((av) < (1 << 8) \ + ? nbits_table[av] \ + : ((av) < (1L << 16) ? nbits_table[(av) >> 8] + 8 \ + : ((av) < (1L << 24) ? nbits_table[(av) >> 16] + 16 : nbits_table[(av) >> 24] + 24))) +#endif + +void init_words(WavpackStream *wps); +void write_entropy_vars(WavpackStream *wps, WavpackMetadata *wpmd); +void write_hybrid_profile(WavpackStream *wps, WavpackMetadata *wpmd); +int read_entropy_vars(WavpackStream *wps, WavpackMetadata *wpmd); +int read_hybrid_profile(WavpackStream *wps, WavpackMetadata *wpmd); +int32_t FASTCALL send_word(WavpackStream *wps, int32_t value, int chan); +void send_words_lossless(WavpackStream *wps, int32_t *buffer, int32_t nsamples); +int32_t FASTCALL get_word(WavpackStream *wps, int chan, int32_t *correction); +int32_t get_words_lossless(WavpackStream *wps, int32_t *buffer, int32_t nsamples); +void flush_word(WavpackStream *wps); +int32_t nosend_word(WavpackStream *wps, int32_t value, int chan); +void scan_word(WavpackStream *wps, int32_t *samples, uint32_t num_samples, int dir); +void update_error_limit(WavpackStream *wps); + +extern const uint32_t bitset[32]; +extern const uint32_t bitmask[32]; +extern const char nbits_table[256]; + +int wp_log2s(int32_t value); +int32_t wp_exp2s(int log); +int FASTCALL wp_log2(uint32_t avalue); + +#ifdef OPT_ASM_X86 +#define LOG2BUFFER log2buffer_x86 +#elif defined(OPT_ASM_X64) && (defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW64__)) +#define LOG2BUFFER log2buffer_x64win +#elif defined(OPT_ASM_X64) +#define LOG2BUFFER log2buffer_x64 +#else +#define LOG2BUFFER log2buffer +#endif + +uint32_t LOG2BUFFER(int32_t *samples, uint32_t num_samples, int limit); + +signed char store_weight(int weight); +int restore_weight(signed char weight); + +#define WORD_EOF ((int32_t)(1L << 31)) + +void WavpackFloatNormalize(int32_t *values, int32_t num_values, int delta_exp); + +/////////////////////////// high-level unpacking API and support //////////////////////////// +// modules: open_utils.c, unpack_utils.c, unpack_seek.c, unpack_floats.c + +WavpackContext *WavpackOpenFileInputEx64(WavpackStreamReader64 *reader, void *wv_id, void *wvc_id, char *error, + int flags, int norm_offset); +WavpackContext *WavpackOpenFileInputEx(WavpackStreamReader *reader, void *wv_id, void *wvc_id, char *error, int flags, + int norm_offset); +WavpackContext *WavpackOpenFileInput(const char *infilename, char *error, int flags, int norm_offset); + +#define OPEN_WVC 0x1 // open/read "correction" file +#define OPEN_TAGS 0x2 // read ID3v1 / APEv2 tags (seekable file) +#define OPEN_WRAPPER 0x4 // make audio wrapper available (i.e. RIFF) +#define OPEN_2CH_MAX 0x8 // open multichannel as stereo (no downmix) +#define OPEN_NORMALIZE 0x10 // normalize floating point data to +/- 1.0 +#define OPEN_STREAMING 0x20 // "streaming" mode blindly unpacks blocks +// w/o regard to header file position info +#define OPEN_EDIT_TAGS 0x40 // allow editing of tags +#define OPEN_FILE_UTF8 0x80 // assume filenames are UTF-8 encoded, not ANSI (Windows only) + +// new for version 5 + +#define OPEN_DSD_NATIVE 0x100 // open DSD files as bitstreams +// (returned as 8-bit "samples" stored in 32-bit words) +#define OPEN_DSD_AS_PCM 0x200 // open DSD files as 24-bit PCM (decimated 8x) +#define OPEN_ALT_TYPES 0x400 // application is aware of alternate file types & qmode +// (just affects retrieving wrappers & MD5 checksums) +#define OPEN_NO_CHECKSUM 0x800 // don't verify block checksums before decoding + +int WavpackGetMode(WavpackContext *wpc); + +#define MODE_WVC 0x1 +#define MODE_LOSSLESS 0x2 +#define MODE_HYBRID 0x4 +#define MODE_FLOAT 0x8 +#define MODE_VALID_TAG 0x10 +#define MODE_HIGH 0x20 +#define MODE_FAST 0x40 +#define MODE_EXTRA 0x80 // extra mode used, see MODE_XMODE for possible level +#define MODE_APETAG 0x100 +#define MODE_SFX 0x200 +#define MODE_VERY_HIGH 0x400 +#define MODE_MD5 0x800 +#define MODE_XMODE 0x7000 // mask for extra level (1-6, 0=unknown) +#define MODE_DNS 0x8000 + +int WavpackGetQualifyMode(WavpackContext *wpc); +int WavpackGetVersion(WavpackContext *wpc); +uint32_t WavpackUnpackSamples(WavpackContext *wpc, int32_t *buffer, uint32_t samples); +int WavpackSeekSample(WavpackContext *wpc, uint32_t sample); +int WavpackSeekSample64(WavpackContext *wpc, int64_t sample); +int WavpackGetMD5Sum(WavpackContext *wpc, unsigned char data[16]); + +int WavpackVerifySingleBlock(unsigned char *buffer, int verify_checksum); +uint32_t read_next_header(WavpackStreamReader64 *reader, void *id, WavpackHeader *wphdr); +int read_wvc_block(WavpackContext *wpc); + +/////////////////////////// high-level packing API and support //////////////////////////// +// modules: pack_utils.c, pack_floats.c + +WavpackContext *WavpackOpenFileOutput(WavpackBlockOutput blockout, void *wv_id, void *wvc_id); +int WavpackSetConfiguration(WavpackContext *wpc, WavpackConfig *config, uint32_t total_samples); +int WavpackSetConfiguration64(WavpackContext *wpc, WavpackConfig *config, int64_t total_samples, + const unsigned char *chan_ids); +int WavpackPackInit(WavpackContext *wpc); +int WavpackAddWrapper(WavpackContext *wpc, void *data, uint32_t bcount); +int WavpackPackSamples(WavpackContext *wpc, int32_t *sample_buffer, uint32_t sample_count); +int WavpackFlushSamples(WavpackContext *wpc); +int WavpackStoreMD5Sum(WavpackContext *wpc, unsigned char data[16]); +void WavpackSeekTrailingWrapper(WavpackContext *wpc); +void WavpackUpdateNumSamples(WavpackContext *wpc, void *first_block); +void *WavpackGetWrapperLocation(void *first_block, uint32_t *size); + +/////////////////////////////////// common utilities //////////////////////////////////// +// module: common_utils.c + +extern const uint32_t sample_rates[16]; +uint32_t WavpackGetLibraryVersion(void); +const char *WavpackGetLibraryVersionString(void); +uint32_t WavpackGetSampleRate(WavpackContext *wpc); +int WavpackGetBitsPerSample(WavpackContext *wpc); +int WavpackGetBytesPerSample(WavpackContext *wpc); +int WavpackGetNumChannels(WavpackContext *wpc); +int WavpackGetChannelMask(WavpackContext *wpc); +int WavpackGetReducedChannels(WavpackContext *wpc); +int WavpackGetFloatNormExp(WavpackContext *wpc); +uint32_t WavpackGetNumSamples(WavpackContext *wpc); +int64_t WavpackGetNumSamples64(WavpackContext *wpc); +uint32_t WavpackGetSampleIndex(WavpackContext *wpc); +int64_t WavpackGetSampleIndex64(WavpackContext *wpc); +char *WavpackGetErrorMessage(WavpackContext *wpc); +int WavpackGetNumErrors(WavpackContext *wpc); +int WavpackLossyBlocks(WavpackContext *wpc); +uint32_t WavpackGetWrapperBytes(WavpackContext *wpc); +unsigned char *WavpackGetWrapperData(WavpackContext *wpc); +void WavpackFreeWrapper(WavpackContext *wpc); +double WavpackGetProgress(WavpackContext *wpc); +uint32_t WavpackGetFileSize(WavpackContext *wpc); +int64_t WavpackGetFileSize64(WavpackContext *wpc); +double WavpackGetRatio(WavpackContext *wpc); +double WavpackGetAverageBitrate(WavpackContext *wpc, int count_wvc); +double WavpackGetInstantBitrate(WavpackContext *wpc); +WavpackContext *WavpackCloseFile(WavpackContext *wpc); +void WavpackLittleEndianToNative(void *data, char *format); +void WavpackNativeToLittleEndian(void *data, char *format); +void WavpackBigEndianToNative(void *data, char *format); +void WavpackNativeToBigEndian(void *data, char *format); + +void free_streams(WavpackContext *wpc); + +/////////////////////////////////// tag utilities //////////////////////////////////// +// modules: tags.c, tag_utils.c + +int WavpackGetNumTagItems(WavpackContext *wpc); +int WavpackGetTagItem(WavpackContext *wpc, const char *item, char *value, int size); +int WavpackGetTagItemIndexed(WavpackContext *wpc, int index, char *item, int size); +int WavpackGetNumBinaryTagItems(WavpackContext *wpc); +int WavpackGetBinaryTagItem(WavpackContext *wpc, const char *item, char *value, int size); +int WavpackGetBinaryTagItemIndexed(WavpackContext *wpc, int index, char *item, int size); +int WavpackAppendTagItem(WavpackContext *wpc, const char *item, const char *value, int vsize); +int WavpackAppendBinaryTagItem(WavpackContext *wpc, const char *item, const char *value, int vsize); +int WavpackDeleteTagItem(WavpackContext *wpc, const char *item); +int WavpackWriteTag(WavpackContext *wpc); +int load_tag(WavpackContext *wpc); +void free_tag(M_Tag *m_tag); +int valid_tag(M_Tag *m_tag); +int editable_tag(M_Tag *m_tag); + +#endif diff --git a/wavpack/wavpack_version.h b/wavpack/wavpack_version.h new file mode 100644 index 0000000..6e9139e --- /dev/null +++ b/wavpack/wavpack_version.h @@ -0,0 +1,19 @@ +//////////////////////////////////////////////////////////////////////////// +// **** WAVPACK **** // +// Hybrid Lossless Wavefile Compressor // +// Copyright (c) 1998 - 2006 Conifer Software. // +// All Rights Reserved. // +// Distributed under the BSD Software License (see license.txt) // +//////////////////////////////////////////////////////////////////////////// + +// wavpack_version.h + +#ifndef WAVPACK_VERSION_H +#define WAVPACK_VERSION_H + +#define LIBWAVPACK_MAJOR 5 +#define LIBWAVPACK_MINOR 1 +#define LIBWAVPACK_MICRO 0 +#define LIBWAVPACK_VERSION_STRING "5.1.0" + +#endif diff --git a/winzipjpeg/ArithmeticDecoder.c b/winzipjpeg/ArithmeticDecoder.c new file mode 100644 index 0000000..3e55e17 --- /dev/null +++ b/winzipjpeg/ArithmeticDecoder.c @@ -0,0 +1,696 @@ +/* + * ArithmeticDecoder.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "ArithmeticDecoder.h" + +// Arithmetic decoder based on US patent 4791403. + +static void InitDec(WinZipJPEGArithmeticDecoder *self); + +static unsigned int LogDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); + +static void UpdateMPS(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); +static void QSmaller(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); + +static void UpdateLPS(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); +static void QBigger(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); +static void IncrIndex(int *i, int *incrsv); +static void DblIndex(int *i, int *incrsv); + +static void LRMBig(WinZipJPEGArithmeticDecoder *self); +static void Renorm(WinZipJPEGArithmeticDecoder *self); +static uint8_t ByteIn(WinZipJPEGArithmeticDecoder *self); + +static uint32_t AntilogX(int16_t lr); +static int16_t LogX(uint32_t x); + +static uint16_t logp[]; // log p at index i +static uint16_t lqp[]; // log q - log p +static uint16_t nmaxlp[]; // nmax * lp +static uint8_t halfi[]; // pointer to q halved +static uint8_t dbli[]; // pointer to q doubled + +static uint16_t alogtbl[]; +static uint16_t logtbl[]; +static uint8_t chartbl[]; + +void InitializeWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGReadFunction *readfunc, + void *inputcontext) +{ + self->readfunc = readfunc; + self->inputcontext = inputcontext; + self->eof = false; + + InitDec(self); +} + +void InitializeWinZipJPEGContext(WinZipJPEGContext *self) +{ + self->dlrm = nmaxlp[0]; + self->i = 0; + self->k = 0; + self->mps = 0; +} + +void InitializeWinZipJPEGContexts(WinZipJPEGContext *first, size_t bytes) +{ + for(int i = 0; i < bytes / sizeof(WinZipJPEGContext); i++) InitializeWinZipJPEGContext(first + i); +} + +void InitializeFixedWinZipJPEGContext(WinZipJPEGContext *self) +{ + self->dlrm = nmaxlp[0]; + self->i = 48; + self->k = 0; + self->mps = 0; +} + +int NextBitFromWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + self->dx = 0; // Otherwise tests don't pass. + int bit = LogDecoder(self, context); + self->lp = logp[context->i]; + return bit; +} + +void FlushWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self) +{ + Renorm(self); + + if(self->currbyte == 0xff && self->lastbyte == 0xff) ByteIn(self); +} + +static void InitDec(WinZipJPEGArithmeticDecoder *self) +{ + self->kmin2 = 0; + self->kmin1 = 1; + self->kmin = 5; + self->kmax = 11; + + uint8_t b1 = ByteIn(self); + uint8_t b2 = ByteIn(self); + self->x = (b1 << 8) | b2; + + self->lr = 0x1001; + self->lrm = self->lr; + self->lx = LogX(self->x); + + if(self->x == 0xffff) ByteIn(self); +} + +static unsigned int LogDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + self->lrm = self->lr + context->dlrm; + LRMBig(self); + + self->lr += logp[context->i]; + + unsigned int bit = context->mps; + + int lrt; + if(self->lx < self->lrm) + lrt = self->lx; + else + lrt = self->lrm; + + if(self->lr >= lrt) + { + if(self->lr < self->lx) { UpdateMPS(self, context); } + else + { + Renorm(self); + if(self->lr < self->lx) + { + if(self->lr >= self->lrm) UpdateMPS(self, context); + } + else + { + bit ^= 1; + + context->k++; + + uint32_t dx = AntilogX(self->lr); + self->x -= dx; + self->lx = LogX(self->x); + self->dx = dx; // for tests + + UpdateLPS(self, context); + } + } + } + + context->dlrm = self->lrm - self->lr; + + return bit; +} + +static void UpdateMPS(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + if(context->k <= self->kmin) QSmaller(self, context); + context->k = 0; + self->lrm = self->lr + nmaxlp[context->i]; + LRMBig(self); +} + +static void QSmaller(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + if(context->i >= 47) return; // WinZip modification. + + context->i++; + + if(context->k <= self->kmin1) + { + context->i += halfi[context->i]; + if(context->k <= self->kmin2) { context->i += halfi[context->i]; } + } +} + +static void UpdateLPS(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + self->lr += lqp[context->i]; + self->lrm += lqp[context->i]; + + if(context->k >= self->kmax) + { + QBigger(self, context); + context->k = 0; + self->lrm = self->lr + nmaxlp[context->i]; + } + else + { + if(self->lrm < self->lr) self->lrm = self->lr; + } +} + +static void QBigger(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context) +{ + if(context->i >= 48) return; // WinZip modification. + + int32_t dlrm = self->lrm - self->lr; + int incrsv = 0; + + if(dlrm >= nmaxlp[context->i] / 2) + { + dlrm = nmaxlp[context->i] - dlrm; + if(dlrm <= nmaxlp[context->i] / 4) DblIndex(&context->i, &incrsv); + DblIndex(&context->i, &incrsv); + } + else + { + if(dlrm >= nmaxlp[context->i] / 4) IncrIndex(&context->i, &incrsv); + IncrIndex(&context->i, &incrsv); + } + + if(context->i <= 0) + { + context->i = incrsv; + context->mps = context->mps ^ 1; + } + + self->lrm = self->lr + dlrm; +} + +static void IncrIndex(int *i, int *incrsv) +{ + if(*i > 0) + (*i)--; + else + (*incrsv)++; +} + +static void DblIndex(int *i, int *incrsv) +{ + if(*i > 0) + *i -= dbli[*i]; + else + *incrsv += dbli[*i]; +} + +static void LRMBig(WinZipJPEGArithmeticDecoder *self) +{ + if(self->lrm > 0x7ff) Renorm(self); +} + +static void Renorm(WinZipJPEGArithmeticDecoder *self) +{ + while(self->lr > 0x1fff) + { + if(self->currbyte == 0xff && self->lastbyte == 0xff) { self->x += ByteIn(self); } + self->x = (self->x << 8) | ByteIn(self); + self->lr -= 0x2000; + self->lrm -= 0x2000; + } + + self->lx = LogX(self->x); +} + +static uint8_t ByteIn(WinZipJPEGArithmeticDecoder *self) +{ + self->lastbyte = self->currbyte; + + if(self->readfunc(self->inputcontext, &self->currbyte, 1) != 1) + { + self->eof = true; + self->currbyte = 0; + } + + return self->currbyte; +} + +static int16_t LogX(uint32_t x) +{ + unsigned int highbits = x >> 12; + if(highbits == 0) return 0x2000; + + int whole; + if(highbits < 512) + whole = chartbl[highbits]; + else + whole = 0; + + int shift = 8 - whole; + + int negfraction; + if(shift >= 0) + negfraction = logtbl[(x >> shift) & 0xfff]; + else + negfraction = logtbl[(x << -shift) & 0xfff]; // Is this necessary? No idea. + + return (whole << 10) - negfraction; +} + +static uint32_t AntilogX(int16_t lr) +{ + int whole = lr >> 10; + unsigned int fraction = lr & 0x3ff; + + int shift = 7 - whole; + if(shift >= 0) + return alogtbl[fraction] << shift; + else + return alogtbl[fraction] >> -shift; // Is this necessary? No idea. +} + +static uint16_t logp[49] = // log p at index i + { + 1024, 895, 795, 706, 628, 559, 493, 437, 379, 331, 287, 247, 212, 186, 158, 143, 127, + 110, 98, 84, 72, 65, 59, 53, 48, 45, 42, 40, 37, 35, 33, 30, 28, 26, + 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 4, 3, 2, 1, 1024, +}; + +static uint16_t lqp[49] = // log q - log p + { + 0, 272, 502, 726, 941, 1150, 1371, 1578, 1819, 2044, 2278, 2521, 2765, 2971, 3227, 3382, 3566, + 3788, 3965, 4200, 4435, 4590, 4737, 4899, 5050, 5147, 5250, 5325, 5441, 5527, 5617, 5758, 5863, 5976, + 6157, 6295, 6447, 6616, 6806, 7024, 7278, 7585, 7972, 8495, 8884, 9309, 10065, 11689, 0, +}; + +static uint16_t nmaxlp[49] = // nmax * lp + { + 16384, 16110, 15105, 14826, 14444, 13975, 13804, 13547, 13265, 13240, 12915, 12844, 12720, + 12648, 12482, 12441, 12319, 12320, 12250, 12180, 12168, 12155, 12154, 12084, 12096, 12105, + 12096, 12080, 12062, 12075, 12078, 12060, 12068, 12090, 12075, 12075, 12103, 12121, 12150, + 12181, 12221, 12294, 12411, 12615, 13120, 13113, 14574, 21860, 0, +}; + +static uint8_t halfi[49] = // pointer to q halved + { + 8, 8, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, 9, 10, 10, 10, 10, + 10, 10, 9, 9, 8, 8, 7, 7, 6, 6, 6, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 1, 0, 0, +}; + +static uint8_t dbli[49] = // pointer to q doubled + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 8, 8, 9, 9, 10, 10, 10, 10, 10, 10, 9, 8, 7, 6, 6, 5, 4, 3, 3, 3, 2, 1, 0, +}; + +static uint16_t alogtbl[1024] = { + 0x2000, 0x1ffd, 0x1ff7, 0x1ff1, 0x1fec, 0x1fe6, 0x1fe1, 0x1fdb, 0x1fd6, 0x1fd0, 0x1fcb, 0x1fc5, 0x1fc0, 0x1fba, + 0x1fb5, 0x1faf, 0x1faa, 0x1fa4, 0x1f9f, 0x1f99, 0x1f94, 0x1f8e, 0x1f89, 0x1f83, 0x1f7e, 0x1f79, 0x1f73, 0x1f6e, + 0x1f68, 0x1f63, 0x1f5d, 0x1f58, 0x1f53, 0x1f4d, 0x1f48, 0x1f42, 0x1f3d, 0x1f37, 0x1f32, 0x1f2d, 0x1f27, 0x1f22, + 0x1f1c, 0x1f17, 0x1f12, 0x1f0c, 0x1f07, 0x1f02, 0x1efc, 0x1ef7, 0x1ef1, 0x1eec, 0x1ee7, 0x1ee1, 0x1edc, 0x1ed7, + 0x1ed1, 0x1ecc, 0x1ec7, 0x1ec1, 0x1ebc, 0x1eb7, 0x1eb1, 0x1eac, 0x1ea7, 0x1ea1, 0x1e9c, 0x1e97, 0x1e92, 0x1e8c, + 0x1e87, 0x1e82, 0x1e7c, 0x1e77, 0x1e72, 0x1e6d, 0x1e67, 0x1e62, 0x1e5d, 0x1e58, 0x1e52, 0x1e4d, 0x1e48, 0x1e43, + 0x1e3d, 0x1e38, 0x1e33, 0x1e2e, 0x1e28, 0x1e23, 0x1e1e, 0x1e19, 0x1e14, 0x1e0e, 0x1e09, 0x1e04, 0x1dff, 0x1dfa, + 0x1df4, 0x1def, 0x1dea, 0x1de5, 0x1de0, 0x1dda, 0x1dd5, 0x1dd0, 0x1dcb, 0x1dc6, 0x1dc1, 0x1dbc, 0x1db6, 0x1db1, + 0x1dac, 0x1da7, 0x1da2, 0x1d9d, 0x1d98, 0x1d93, 0x1d8d, 0x1d88, 0x1d83, 0x1d7e, 0x1d79, 0x1d74, 0x1d6f, 0x1d6a, + 0x1d65, 0x1d5f, 0x1d5a, 0x1d55, 0x1d50, 0x1d4b, 0x1d46, 0x1d41, 0x1d3c, 0x1d37, 0x1d32, 0x1d2d, 0x1d28, 0x1d23, + 0x1d1e, 0x1d19, 0x1d14, 0x1d0e, 0x1d09, 0x1d04, 0x1cff, 0x1cfa, 0x1cf5, 0x1cf0, 0x1ceb, 0x1ce6, 0x1ce1, 0x1cdc, + 0x1cd7, 0x1cd2, 0x1ccd, 0x1cc8, 0x1cc3, 0x1cbf, 0x1cba, 0x1cb5, 0x1cb0, 0x1cab, 0x1ca6, 0x1ca1, 0x1c9c, 0x1c97, + 0x1c92, 0x1c8d, 0x1c88, 0x1c83, 0x1c7e, 0x1c79, 0x1c74, 0x1c6f, 0x1c6a, 0x1c65, 0x1c61, 0x1c5c, 0x1c57, 0x1c52, + 0x1c4d, 0x1c48, 0x1c43, 0x1c3e, 0x1c39, 0x1c34, 0x1c2f, 0x1c2b, 0x1c26, 0x1c21, 0x1c1c, 0x1c17, 0x1c12, 0x1c0d, + 0x1c09, 0x1c04, 0x1bff, 0x1bfa, 0x1bf5, 0x1bf0, 0x1bec, 0x1be7, 0x1be2, 0x1bdd, 0x1bd8, 0x1bd3, 0x1bcf, 0x1bca, + 0x1bc5, 0x1bc0, 0x1bbb, 0x1bb7, 0x1bb2, 0x1bad, 0x1ba8, 0x1ba3, 0x1b9f, 0x1b9a, 0x1b95, 0x1b90, 0x1b8c, 0x1b87, + 0x1b82, 0x1b7d, 0x1b78, 0x1b74, 0x1b6f, 0x1b6a, 0x1b66, 0x1b61, 0x1b5c, 0x1b57, 0x1b53, 0x1b4e, 0x1b49, 0x1b44, + 0x1b40, 0x1b3b, 0x1b36, 0x1b32, 0x1b2d, 0x1b28, 0x1b23, 0x1b1f, 0x1b1a, 0x1b15, 0x1b11, 0x1b0c, 0x1b07, 0x1b03, + 0x1afe, 0x1af9, 0x1af5, 0x1af0, 0x1aeb, 0x1ae7, 0x1ae2, 0x1add, 0x1ad9, 0x1ad4, 0x1acf, 0x1acb, 0x1ac6, 0x1ac1, + 0x1abd, 0x1ab8, 0x1ab4, 0x1aaf, 0x1aaa, 0x1aa6, 0x1aa1, 0x1a9c, 0x1a98, 0x1a93, 0x1a8f, 0x1a8a, 0x1a85, 0x1a81, + 0x1a7c, 0x1a78, 0x1a73, 0x1a6f, 0x1a6a, 0x1a65, 0x1a61, 0x1a5c, 0x1a58, 0x1a53, 0x1a4f, 0x1a4a, 0x1a46, 0x1a41, + 0x1a3c, 0x1a38, 0x1a33, 0x1a2f, 0x1a2a, 0x1a26, 0x1a21, 0x1a1d, 0x1a18, 0x1a14, 0x1a0f, 0x1a0b, 0x1a06, 0x1a02, + 0x19fd, 0x19f9, 0x19f4, 0x19f0, 0x19eb, 0x19e7, 0x19e2, 0x19de, 0x19d9, 0x19d5, 0x19d0, 0x19cc, 0x19c7, 0x19c3, + 0x19be, 0x19ba, 0x19b6, 0x19b1, 0x19ad, 0x19a8, 0x19a4, 0x199f, 0x199b, 0x1996, 0x1992, 0x198e, 0x1989, 0x1985, + 0x1980, 0x197c, 0x1978, 0x1973, 0x196f, 0x196a, 0x1966, 0x1962, 0x195d, 0x1959, 0x1954, 0x1950, 0x194c, 0x1947, + 0x1943, 0x193e, 0x193a, 0x1936, 0x1931, 0x192d, 0x1929, 0x1924, 0x1920, 0x191c, 0x1917, 0x1913, 0x190f, 0x190a, + 0x1906, 0x1902, 0x18fd, 0x18f9, 0x18f5, 0x18f0, 0x18ec, 0x18e8, 0x18e3, 0x18df, 0x18db, 0x18d6, 0x18d2, 0x18ce, + 0x18ca, 0x18c5, 0x18c1, 0x18bd, 0x18b8, 0x18b4, 0x18b0, 0x18ac, 0x18a7, 0x18a3, 0x189f, 0x189b, 0x1896, 0x1892, + 0x188e, 0x188a, 0x1885, 0x1881, 0x187d, 0x1879, 0x1874, 0x1870, 0x186c, 0x1868, 0x1863, 0x185f, 0x185b, 0x1857, + 0x1853, 0x184e, 0x184a, 0x1846, 0x1842, 0x183e, 0x1839, 0x1835, 0x1831, 0x182d, 0x1829, 0x1824, 0x1820, 0x181c, + 0x1818, 0x1814, 0x180f, 0x180b, 0x1807, 0x1803, 0x17ff, 0x17fb, 0x17f7, 0x17f2, 0x17ee, 0x17ea, 0x17e6, 0x17e2, + 0x17de, 0x17da, 0x17d5, 0x17d1, 0x17cd, 0x17c9, 0x17c5, 0x17c1, 0x17bd, 0x17b9, 0x17b5, 0x17b0, 0x17ac, 0x17a8, + 0x17a4, 0x17a0, 0x179c, 0x1798, 0x1794, 0x1790, 0x178c, 0x1788, 0x1784, 0x177f, 0x177b, 0x1777, 0x1773, 0x176f, + 0x176b, 0x1767, 0x1763, 0x175f, 0x175b, 0x1757, 0x1753, 0x174f, 0x174b, 0x1747, 0x1743, 0x173f, 0x173b, 0x1737, + 0x1733, 0x172f, 0x172b, 0x1727, 0x1723, 0x171f, 0x171b, 0x1717, 0x1713, 0x170f, 0x170b, 0x1707, 0x1703, 0x16ff, + 0x16fb, 0x16f7, 0x16f3, 0x16ef, 0x16eb, 0x16e7, 0x16e3, 0x16df, 0x16db, 0x16d7, 0x16d3, 0x16cf, 0x16cb, 0x16c7, + 0x16c3, 0x16bf, 0x16bb, 0x16b7, 0x16b3, 0x16b0, 0x16ac, 0x16a8, 0x16a4, 0x16a0, 0x169c, 0x1698, 0x1694, 0x1690, + 0x168c, 0x1688, 0x1685, 0x1681, 0x167d, 0x1679, 0x1675, 0x1671, 0x166d, 0x1669, 0x1665, 0x1662, 0x165e, 0x165a, + 0x1656, 0x1652, 0x164e, 0x164a, 0x1647, 0x1643, 0x163f, 0x163b, 0x1637, 0x1633, 0x162f, 0x162c, 0x1628, 0x1624, + 0x1620, 0x161c, 0x1618, 0x1615, 0x1611, 0x160d, 0x1609, 0x1605, 0x1602, 0x15fe, 0x15fa, 0x15f6, 0x15f2, 0x15ee, + 0x15eb, 0x15e7, 0x15e3, 0x15df, 0x15dc, 0x15d8, 0x15d4, 0x15d0, 0x15cc, 0x15c9, 0x15c5, 0x15c1, 0x15bd, 0x15b9, + 0x15b6, 0x15b2, 0x15ae, 0x15aa, 0x15a7, 0x15a3, 0x159f, 0x159c, 0x1598, 0x1594, 0x1590, 0x158d, 0x1589, 0x1585, + 0x1581, 0x157e, 0x157a, 0x1576, 0x1573, 0x156f, 0x156b, 0x1567, 0x1564, 0x1560, 0x155c, 0x1559, 0x1555, 0x1551, + 0x154e, 0x154a, 0x1546, 0x1542, 0x153f, 0x153b, 0x1537, 0x1534, 0x1530, 0x152c, 0x1529, 0x1525, 0x1521, 0x151e, + 0x151a, 0x1516, 0x1513, 0x150f, 0x150b, 0x1508, 0x1504, 0x1501, 0x14fd, 0x14f9, 0x14f6, 0x14f2, 0x14ee, 0x14eb, + 0x14e7, 0x14e4, 0x14e0, 0x14dc, 0x14d9, 0x14d5, 0x14d2, 0x14ce, 0x14ca, 0x14c7, 0x14c3, 0x14c0, 0x14bc, 0x14b8, + 0x14b5, 0x14b1, 0x14ae, 0x14aa, 0x14a6, 0x14a3, 0x149f, 0x149c, 0x1498, 0x1495, 0x1491, 0x148d, 0x148a, 0x1486, + 0x1483, 0x147f, 0x147c, 0x1478, 0x1475, 0x1471, 0x146e, 0x146a, 0x1466, 0x1463, 0x145f, 0x145c, 0x1458, 0x1455, + 0x1451, 0x144e, 0x144a, 0x1447, 0x1443, 0x1440, 0x143c, 0x1439, 0x1435, 0x1432, 0x142e, 0x142b, 0x1427, 0x1424, + 0x1420, 0x141d, 0x1419, 0x1416, 0x1412, 0x140f, 0x140b, 0x1408, 0x1405, 0x1401, 0x13fd, 0x13fa, 0x13f7, 0x13f3, + 0x13f0, 0x13ec, 0x13e9, 0x13e5, 0x13e2, 0x13de, 0x13db, 0x13d8, 0x13d4, 0x13d1, 0x13cd, 0x13ca, 0x13c7, 0x13c3, + 0x13c0, 0x13bc, 0x13b9, 0x13b5, 0x13b2, 0x13ae, 0x13ab, 0x13a8, 0x13a4, 0x13a1, 0x139e, 0x139a, 0x1397, 0x1393, + 0x1390, 0x138d, 0x1389, 0x1386, 0x1382, 0x137f, 0x137c, 0x1378, 0x1375, 0x1372, 0x136e, 0x136b, 0x1367, 0x1364, + 0x1361, 0x135d, 0x135a, 0x1357, 0x1353, 0x1350, 0x134d, 0x1349, 0x1346, 0x1343, 0x133f, 0x133c, 0x1339, 0x1335, + 0x1332, 0x132f, 0x132b, 0x1328, 0x1325, 0x1321, 0x131e, 0x131b, 0x1317, 0x1314, 0x1311, 0x130e, 0x130a, 0x1307, + 0x1304, 0x1300, 0x12fd, 0x12fa, 0x12f7, 0x12f3, 0x12f0, 0x12ed, 0x12e9, 0x12e6, 0x12e3, 0x12df, 0x12dc, 0x12d9, + 0x12d6, 0x12d2, 0x12cf, 0x12cc, 0x12c9, 0x12c5, 0x12c2, 0x12bf, 0x12bc, 0x12b8, 0x12b5, 0x12b2, 0x12af, 0x12ac, + 0x12a8, 0x12a5, 0x12a2, 0x129f, 0x129b, 0x1298, 0x1295, 0x1292, 0x128e, 0x128b, 0x1288, 0x1285, 0x1282, 0x127e, + 0x127b, 0x1278, 0x1275, 0x1272, 0x126e, 0x126b, 0x1268, 0x1265, 0x1262, 0x125e, 0x125b, 0x1258, 0x1255, 0x1252, + 0x124f, 0x124b, 0x1248, 0x1245, 0x1242, 0x123f, 0x123c, 0x1238, 0x1235, 0x1232, 0x122f, 0x122c, 0x1229, 0x1226, + 0x1222, 0x121f, 0x121c, 0x1219, 0x1216, 0x1213, 0x1210, 0x120c, 0x1209, 0x1206, 0x1203, 0x1200, 0x11fd, 0x11fa, + 0x11f7, 0x11f4, 0x11f0, 0x11ed, 0x11ea, 0x11e7, 0x11e4, 0x11e1, 0x11de, 0x11db, 0x11d8, 0x11d5, 0x11d1, 0x11ce, + 0x11cb, 0x11c8, 0x11c5, 0x11c2, 0x11bf, 0x11bc, 0x11b9, 0x11b6, 0x11b3, 0x11b0, 0x11ad, 0x11a9, 0x11a6, 0x11a3, + 0x11a0, 0x119d, 0x119a, 0x1197, 0x1194, 0x1191, 0x118e, 0x118b, 0x1188, 0x1185, 0x1182, 0x117f, 0x117c, 0x1179, + 0x1176, 0x1173, 0x1170, 0x116d, 0x116a, 0x1167, 0x1164, 0x1161, 0x115e, 0x115b, 0x1158, 0x1155, 0x1152, 0x114f, + 0x114c, 0x1149, 0x1146, 0x1143, 0x1140, 0x113d, 0x113a, 0x1137, 0x1134, 0x1131, 0x112e, 0x112b, 0x1128, 0x1125, + 0x1122, 0x111f, 0x111c, 0x1119, 0x1116, 0x1113, 0x1110, 0x110d, 0x110a, 0x1107, 0x1104, 0x1101, 0x10fe, 0x10fb, + 0x10f8, 0x10f5, 0x10f2, 0x10ef, 0x10ec, 0x10e9, 0x10e6, 0x10e3, 0x10e0, 0x10de, 0x10db, 0x10d8, 0x10d5, 0x10d2, + 0x10cf, 0x10cc, 0x10c9, 0x10c6, 0x10c3, 0x10c0, 0x10bd, 0x10ba, 0x10b8, 0x10b5, 0x10b2, 0x10af, 0x10ac, 0x10a9, + 0x10a6, 0x10a3, 0x10a0, 0x109e, 0x109b, 0x1098, 0x1095, 0x1092, 0x108f, 0x108c, 0x1089, 0x1086, 0x1084, 0x1081, + 0x107e, 0x107b, 0x1078, 0x1075, 0x1072, 0x1070, 0x106d, 0x106a, 0x1067, 0x1064, 0x1061, 0x105e, 0x105c, 0x1059, + 0x1056, 0x1053, 0x1050, 0x104d, 0x104a, 0x1048, 0x1045, 0x1042, 0x103f, 0x103c, 0x1039, 0x1037, 0x1034, 0x1031, + 0x102e, 0x102b, 0x1028, 0x1026, 0x1023, 0x1020, 0x101d, 0x101a, 0x1018, 0x1015, 0x1012, 0x100f, 0x100c, 0x100a, + 0x1007, 0x1004, +}; + +static uint16_t logtbl[4096] = { + 0x000, 0x000, 0x000, 0x000, 0x001, 0x001, 0x001, 0x002, 0x002, 0x002, 0x003, 0x003, 0x004, 0x004, 0x004, 0x005, + 0x005, 0x005, 0x006, 0x006, 0x006, 0x007, 0x007, 0x007, 0x008, 0x008, 0x009, 0x009, 0x009, 0x00a, 0x00a, 0x00a, + 0x00b, 0x00b, 0x00b, 0x00c, 0x00c, 0x00c, 0x00d, 0x00d, 0x00e, 0x00e, 0x00e, 0x00f, 0x00f, 0x00f, 0x010, 0x010, + 0x010, 0x011, 0x011, 0x011, 0x012, 0x012, 0x012, 0x013, 0x013, 0x014, 0x014, 0x014, 0x015, 0x015, 0x015, 0x016, + 0x016, 0x016, 0x017, 0x017, 0x017, 0x018, 0x018, 0x018, 0x019, 0x019, 0x01a, 0x01a, 0x01a, 0x01b, 0x01b, 0x01b, + 0x01c, 0x01c, 0x01c, 0x01d, 0x01d, 0x01d, 0x01e, 0x01e, 0x01e, 0x01f, 0x01f, 0x01f, 0x020, 0x020, 0x021, 0x021, + 0x021, 0x022, 0x022, 0x022, 0x023, 0x023, 0x023, 0x024, 0x024, 0x024, 0x025, 0x025, 0x025, 0x026, 0x026, 0x026, + 0x027, 0x027, 0x028, 0x028, 0x028, 0x029, 0x029, 0x029, 0x02a, 0x02a, 0x02a, 0x02b, 0x02b, 0x02b, 0x02c, 0x02c, + 0x02c, 0x02d, 0x02d, 0x02d, 0x02e, 0x02e, 0x02f, 0x02f, 0x02f, 0x030, 0x030, 0x030, 0x031, 0x031, 0x031, 0x032, + 0x032, 0x032, 0x033, 0x033, 0x033, 0x034, 0x034, 0x034, 0x035, 0x035, 0x035, 0x036, 0x036, 0x036, 0x037, 0x037, + 0x038, 0x038, 0x038, 0x039, 0x039, 0x039, 0x03a, 0x03a, 0x03a, 0x03b, 0x03b, 0x03b, 0x03c, 0x03c, 0x03c, 0x03d, + 0x03d, 0x03d, 0x03e, 0x03e, 0x03e, 0x03f, 0x03f, 0x03f, 0x040, 0x040, 0x041, 0x041, 0x041, 0x042, 0x042, 0x042, + 0x043, 0x043, 0x043, 0x044, 0x044, 0x044, 0x045, 0x045, 0x045, 0x046, 0x046, 0x046, 0x047, 0x047, 0x047, 0x048, + 0x048, 0x048, 0x049, 0x049, 0x049, 0x04a, 0x04a, 0x04a, 0x04b, 0x04b, 0x04b, 0x04c, 0x04c, 0x04c, 0x04d, 0x04d, + 0x04e, 0x04e, 0x04e, 0x04f, 0x04f, 0x04f, 0x050, 0x050, 0x050, 0x051, 0x051, 0x051, 0x052, 0x052, 0x052, 0x053, + 0x053, 0x053, 0x054, 0x054, 0x054, 0x055, 0x055, 0x055, 0x056, 0x056, 0x056, 0x057, 0x057, 0x057, 0x058, 0x058, + 0x058, 0x059, 0x059, 0x059, 0x05a, 0x05a, 0x05a, 0x05b, 0x05b, 0x05b, 0x05c, 0x05c, 0x05c, 0x05d, 0x05d, 0x05d, + 0x05e, 0x05e, 0x05e, 0x05f, 0x05f, 0x05f, 0x060, 0x060, 0x060, 0x061, 0x061, 0x061, 0x062, 0x062, 0x062, 0x063, + 0x063, 0x063, 0x064, 0x064, 0x064, 0x065, 0x065, 0x065, 0x066, 0x066, 0x066, 0x067, 0x067, 0x067, 0x068, 0x068, + 0x068, 0x069, 0x069, 0x069, 0x06a, 0x06a, 0x06a, 0x06b, 0x06b, 0x06b, 0x06c, 0x06c, 0x06c, 0x06d, 0x06d, 0x06d, + 0x06e, 0x06e, 0x06e, 0x06f, 0x06f, 0x06f, 0x070, 0x070, 0x070, 0x071, 0x071, 0x071, 0x072, 0x072, 0x072, 0x073, + 0x073, 0x073, 0x074, 0x074, 0x074, 0x075, 0x075, 0x075, 0x076, 0x076, 0x076, 0x077, 0x077, 0x077, 0x078, 0x078, + 0x078, 0x079, 0x079, 0x079, 0x07a, 0x07a, 0x07a, 0x07b, 0x07b, 0x07b, 0x07c, 0x07c, 0x07c, 0x07d, 0x07d, 0x07d, + 0x07e, 0x07e, 0x07e, 0x07f, 0x07f, 0x07f, 0x080, 0x080, 0x080, 0x081, 0x081, 0x081, 0x082, 0x082, 0x082, 0x083, + 0x083, 0x083, 0x084, 0x084, 0x084, 0x085, 0x085, 0x085, 0x086, 0x086, 0x086, 0x087, 0x087, 0x087, 0x088, 0x088, + 0x088, 0x089, 0x089, 0x089, 0x08a, 0x08a, 0x08a, 0x08b, 0x08b, 0x08b, 0x08c, 0x08c, 0x08c, 0x08d, 0x08d, 0x08d, + 0x08e, 0x08e, 0x08e, 0x08f, 0x08f, 0x08f, 0x090, 0x090, 0x090, 0x091, 0x091, 0x091, 0x091, 0x092, 0x092, 0x092, + 0x093, 0x093, 0x093, 0x094, 0x094, 0x094, 0x095, 0x095, 0x095, 0x096, 0x096, 0x096, 0x097, 0x097, 0x097, 0x098, + 0x098, 0x098, 0x099, 0x099, 0x099, 0x09a, 0x09a, 0x09a, 0x09b, 0x09b, 0x09b, 0x09c, 0x09c, 0x09c, 0x09d, 0x09d, + 0x09d, 0x09e, 0x09e, 0x09e, 0x09e, 0x09f, 0x09f, 0x09f, 0x0a0, 0x0a0, 0x0a0, 0x0a1, 0x0a1, 0x0a1, 0x0a2, 0x0a2, + 0x0a2, 0x0a3, 0x0a3, 0x0a3, 0x0a4, 0x0a4, 0x0a4, 0x0a5, 0x0a5, 0x0a5, 0x0a6, 0x0a6, 0x0a6, 0x0a7, 0x0a7, 0x0a7, + 0x0a8, 0x0a8, 0x0a8, 0x0a8, 0x0a9, 0x0a9, 0x0a9, 0x0aa, 0x0aa, 0x0aa, 0x0ab, 0x0ab, 0x0ab, 0x0ac, 0x0ac, 0x0ac, + 0x0ad, 0x0ad, 0x0ad, 0x0ae, 0x0ae, 0x0ae, 0x0af, 0x0af, 0x0af, 0x0b0, 0x0b0, 0x0b0, 0x0b1, 0x0b1, 0x0b1, 0x0b1, + 0x0b2, 0x0b2, 0x0b2, 0x0b3, 0x0b3, 0x0b3, 0x0b4, 0x0b4, 0x0b4, 0x0b5, 0x0b5, 0x0b5, 0x0b6, 0x0b6, 0x0b6, 0x0b7, + 0x0b7, 0x0b7, 0x0b8, 0x0b8, 0x0b8, 0x0b8, 0x0b9, 0x0b9, 0x0b9, 0x0ba, 0x0ba, 0x0ba, 0x0bb, 0x0bb, 0x0bb, 0x0bc, + 0x0bc, 0x0bc, 0x0bd, 0x0bd, 0x0bd, 0x0be, 0x0be, 0x0be, 0x0bf, 0x0bf, 0x0bf, 0x0bf, 0x0c0, 0x0c0, 0x0c0, 0x0c1, + 0x0c1, 0x0c1, 0x0c2, 0x0c2, 0x0c2, 0x0c3, 0x0c3, 0x0c3, 0x0c4, 0x0c4, 0x0c4, 0x0c5, 0x0c5, 0x0c5, 0x0c5, 0x0c6, + 0x0c6, 0x0c6, 0x0c7, 0x0c7, 0x0c7, 0x0c8, 0x0c8, 0x0c8, 0x0c9, 0x0c9, 0x0c9, 0x0ca, 0x0ca, 0x0ca, 0x0cb, 0x0cb, + 0x0cb, 0x0cb, 0x0cc, 0x0cc, 0x0cc, 0x0cd, 0x0cd, 0x0cd, 0x0ce, 0x0ce, 0x0ce, 0x0cf, 0x0cf, 0x0cf, 0x0d0, 0x0d0, + 0x0d0, 0x0d0, 0x0d1, 0x0d1, 0x0d1, 0x0d2, 0x0d2, 0x0d2, 0x0d3, 0x0d3, 0x0d3, 0x0d4, 0x0d4, 0x0d4, 0x0d5, 0x0d5, + 0x0d5, 0x0d5, 0x0d6, 0x0d6, 0x0d6, 0x0d7, 0x0d7, 0x0d7, 0x0d8, 0x0d8, 0x0d8, 0x0d9, 0x0d9, 0x0d9, 0x0da, 0x0da, + 0x0da, 0x0da, 0x0db, 0x0db, 0x0db, 0x0dc, 0x0dc, 0x0dc, 0x0dd, 0x0dd, 0x0dd, 0x0de, 0x0de, 0x0de, 0x0de, 0x0df, + 0x0df, 0x0df, 0x0e0, 0x0e0, 0x0e0, 0x0e1, 0x0e1, 0x0e1, 0x0e2, 0x0e2, 0x0e2, 0x0e2, 0x0e3, 0x0e3, 0x0e3, 0x0e4, + 0x0e4, 0x0e4, 0x0e5, 0x0e5, 0x0e5, 0x0e6, 0x0e6, 0x0e6, 0x0e7, 0x0e7, 0x0e7, 0x0e7, 0x0e8, 0x0e8, 0x0e8, 0x0e9, + 0x0e9, 0x0e9, 0x0ea, 0x0ea, 0x0ea, 0x0eb, 0x0eb, 0x0eb, 0x0eb, 0x0ec, 0x0ec, 0x0ec, 0x0ed, 0x0ed, 0x0ed, 0x0ee, + 0x0ee, 0x0ee, 0x0ef, 0x0ef, 0x0ef, 0x0ef, 0x0f0, 0x0f0, 0x0f0, 0x0f1, 0x0f1, 0x0f1, 0x0f2, 0x0f2, 0x0f2, 0x0f3, + 0x0f3, 0x0f3, 0x0f3, 0x0f4, 0x0f4, 0x0f4, 0x0f5, 0x0f5, 0x0f5, 0x0f6, 0x0f6, 0x0f6, 0x0f6, 0x0f7, 0x0f7, 0x0f7, + 0x0f8, 0x0f8, 0x0f8, 0x0f9, 0x0f9, 0x0f9, 0x0f9, 0x0fa, 0x0fa, 0x0fa, 0x0fb, 0x0fb, 0x0fb, 0x0fc, 0x0fc, 0x0fc, + 0x0fd, 0x0fd, 0x0fd, 0x0fd, 0x0fe, 0x0fe, 0x0fe, 0x0ff, 0x0ff, 0x0ff, 0x100, 0x100, 0x100, 0x100, 0x101, 0x101, + 0x101, 0x102, 0x102, 0x102, 0x103, 0x103, 0x103, 0x104, 0x104, 0x104, 0x104, 0x105, 0x105, 0x105, 0x106, 0x106, + 0x106, 0x107, 0x107, 0x107, 0x107, 0x108, 0x108, 0x108, 0x109, 0x109, 0x109, 0x10a, 0x10a, 0x10a, 0x10a, 0x10b, + 0x10b, 0x10b, 0x10c, 0x10c, 0x10c, 0x10d, 0x10d, 0x10d, 0x10d, 0x10e, 0x10e, 0x10e, 0x10f, 0x10f, 0x10f, 0x110, + 0x110, 0x110, 0x110, 0x111, 0x111, 0x111, 0x112, 0x112, 0x112, 0x113, 0x113, 0x113, 0x113, 0x114, 0x114, 0x114, + 0x115, 0x115, 0x115, 0x116, 0x116, 0x116, 0x116, 0x117, 0x117, 0x117, 0x118, 0x118, 0x118, 0x119, 0x119, 0x119, + 0x119, 0x11a, 0x11a, 0x11a, 0x11b, 0x11b, 0x11b, 0x11c, 0x11c, 0x11c, 0x11c, 0x11d, 0x11d, 0x11d, 0x11e, 0x11e, + 0x11e, 0x11e, 0x11f, 0x11f, 0x11f, 0x120, 0x120, 0x120, 0x121, 0x121, 0x121, 0x121, 0x122, 0x122, 0x122, 0x123, + 0x123, 0x123, 0x124, 0x124, 0x124, 0x124, 0x125, 0x125, 0x125, 0x126, 0x126, 0x126, 0x126, 0x127, 0x127, 0x127, + 0x128, 0x128, 0x128, 0x129, 0x129, 0x129, 0x129, 0x12a, 0x12a, 0x12a, 0x12b, 0x12b, 0x12b, 0x12b, 0x12c, 0x12c, + 0x12c, 0x12d, 0x12d, 0x12d, 0x12e, 0x12e, 0x12e, 0x12e, 0x12f, 0x12f, 0x12f, 0x130, 0x130, 0x130, 0x131, 0x131, + 0x131, 0x131, 0x132, 0x132, 0x132, 0x133, 0x133, 0x133, 0x133, 0x134, 0x134, 0x134, 0x135, 0x135, 0x135, 0x135, + 0x136, 0x136, 0x136, 0x137, 0x137, 0x137, 0x137, 0x138, 0x138, 0x138, 0x139, 0x139, 0x139, 0x13a, 0x13a, 0x13a, + 0x13a, 0x13b, 0x13b, 0x13b, 0x13c, 0x13c, 0x13c, 0x13c, 0x13d, 0x13d, 0x13d, 0x13e, 0x13e, 0x13e, 0x13f, 0x13f, + 0x13f, 0x13f, 0x140, 0x140, 0x140, 0x141, 0x141, 0x141, 0x141, 0x142, 0x142, 0x142, 0x143, 0x143, 0x143, 0x143, + 0x144, 0x144, 0x144, 0x145, 0x145, 0x145, 0x145, 0x146, 0x146, 0x146, 0x147, 0x147, 0x147, 0x148, 0x148, 0x148, + 0x148, 0x149, 0x149, 0x149, 0x149, 0x14a, 0x14a, 0x14a, 0x14b, 0x14b, 0x14b, 0x14c, 0x14c, 0x14c, 0x14c, 0x14d, + 0x14d, 0x14d, 0x14e, 0x14e, 0x14e, 0x14e, 0x14f, 0x14f, 0x14f, 0x150, 0x150, 0x150, 0x150, 0x151, 0x151, 0x151, + 0x152, 0x152, 0x152, 0x152, 0x153, 0x153, 0x153, 0x154, 0x154, 0x154, 0x154, 0x155, 0x155, 0x155, 0x156, 0x156, + 0x156, 0x156, 0x157, 0x157, 0x157, 0x158, 0x158, 0x158, 0x158, 0x159, 0x159, 0x159, 0x15a, 0x15a, 0x15a, 0x15a, + 0x15b, 0x15b, 0x15b, 0x15c, 0x15c, 0x15c, 0x15c, 0x15d, 0x15d, 0x15d, 0x15e, 0x15e, 0x15e, 0x15e, 0x15f, 0x15f, + 0x15f, 0x160, 0x160, 0x160, 0x160, 0x161, 0x161, 0x161, 0x162, 0x162, 0x162, 0x162, 0x163, 0x163, 0x163, 0x164, + 0x164, 0x164, 0x164, 0x165, 0x165, 0x165, 0x166, 0x166, 0x166, 0x166, 0x167, 0x167, 0x167, 0x167, 0x168, 0x168, + 0x168, 0x169, 0x169, 0x169, 0x169, 0x16a, 0x16a, 0x16a, 0x16b, 0x16b, 0x16b, 0x16b, 0x16c, 0x16c, 0x16c, 0x16d, + 0x16d, 0x16d, 0x16d, 0x16e, 0x16e, 0x16e, 0x16f, 0x16f, 0x16f, 0x16f, 0x170, 0x170, 0x170, 0x171, 0x171, 0x171, + 0x171, 0x172, 0x172, 0x172, 0x172, 0x173, 0x173, 0x173, 0x174, 0x174, 0x174, 0x174, 0x175, 0x175, 0x175, 0x176, + 0x176, 0x176, 0x176, 0x177, 0x177, 0x177, 0x178, 0x178, 0x178, 0x178, 0x179, 0x179, 0x179, 0x179, 0x17a, 0x17a, + 0x17a, 0x17b, 0x17b, 0x17b, 0x17b, 0x17c, 0x17c, 0x17c, 0x17d, 0x17d, 0x17d, 0x17d, 0x17e, 0x17e, 0x17e, 0x17e, + 0x17f, 0x17f, 0x17f, 0x180, 0x180, 0x180, 0x180, 0x181, 0x181, 0x181, 0x182, 0x182, 0x182, 0x182, 0x183, 0x183, + 0x183, 0x183, 0x184, 0x184, 0x184, 0x185, 0x185, 0x185, 0x185, 0x186, 0x186, 0x186, 0x187, 0x187, 0x187, 0x187, + 0x188, 0x188, 0x188, 0x188, 0x189, 0x189, 0x189, 0x18a, 0x18a, 0x18a, 0x18a, 0x18b, 0x18b, 0x18b, 0x18c, 0x18c, + 0x18c, 0x18c, 0x18d, 0x18d, 0x18d, 0x18d, 0x18e, 0x18e, 0x18e, 0x18f, 0x18f, 0x18f, 0x18f, 0x190, 0x190, 0x190, + 0x190, 0x191, 0x191, 0x191, 0x192, 0x192, 0x192, 0x192, 0x193, 0x193, 0x193, 0x194, 0x194, 0x194, 0x194, 0x195, + 0x195, 0x195, 0x195, 0x196, 0x196, 0x196, 0x197, 0x197, 0x197, 0x197, 0x198, 0x198, 0x198, 0x198, 0x199, 0x199, + 0x199, 0x19a, 0x19a, 0x19a, 0x19a, 0x19b, 0x19b, 0x19b, 0x19b, 0x19c, 0x19c, 0x19c, 0x19d, 0x19d, 0x19d, 0x19d, + 0x19e, 0x19e, 0x19e, 0x19e, 0x19f, 0x19f, 0x19f, 0x1a0, 0x1a0, 0x1a0, 0x1a0, 0x1a1, 0x1a1, 0x1a1, 0x1a1, 0x1a2, + 0x1a2, 0x1a2, 0x1a3, 0x1a3, 0x1a3, 0x1a3, 0x1a4, 0x1a4, 0x1a4, 0x1a4, 0x1a5, 0x1a5, 0x1a5, 0x1a5, 0x1a6, 0x1a6, + 0x1a6, 0x1a7, 0x1a7, 0x1a7, 0x1a7, 0x1a8, 0x1a8, 0x1a8, 0x1a8, 0x1a9, 0x1a9, 0x1a9, 0x1aa, 0x1aa, 0x1aa, 0x1aa, + 0x1ab, 0x1ab, 0x1ab, 0x1ab, 0x1ac, 0x1ac, 0x1ac, 0x1ad, 0x1ad, 0x1ad, 0x1ad, 0x1ae, 0x1ae, 0x1ae, 0x1ae, 0x1af, + 0x1af, 0x1af, 0x1af, 0x1b0, 0x1b0, 0x1b0, 0x1b1, 0x1b1, 0x1b1, 0x1b1, 0x1b2, 0x1b2, 0x1b2, 0x1b2, 0x1b3, 0x1b3, + 0x1b3, 0x1b4, 0x1b4, 0x1b4, 0x1b4, 0x1b5, 0x1b5, 0x1b5, 0x1b5, 0x1b6, 0x1b6, 0x1b6, 0x1b6, 0x1b7, 0x1b7, 0x1b7, + 0x1b8, 0x1b8, 0x1b8, 0x1b8, 0x1b9, 0x1b9, 0x1b9, 0x1b9, 0x1ba, 0x1ba, 0x1ba, 0x1ba, 0x1bb, 0x1bb, 0x1bb, 0x1bc, + 0x1bc, 0x1bc, 0x1bc, 0x1bd, 0x1bd, 0x1bd, 0x1bd, 0x1be, 0x1be, 0x1be, 0x1bf, 0x1bf, 0x1bf, 0x1bf, 0x1c0, 0x1c0, + 0x1c0, 0x1c0, 0x1c1, 0x1c1, 0x1c1, 0x1c1, 0x1c2, 0x1c2, 0x1c2, 0x1c3, 0x1c3, 0x1c3, 0x1c3, 0x1c4, 0x1c4, 0x1c4, + 0x1c4, 0x1c5, 0x1c5, 0x1c5, 0x1c5, 0x1c6, 0x1c6, 0x1c6, 0x1c6, 0x1c7, 0x1c7, 0x1c7, 0x1c8, 0x1c8, 0x1c8, 0x1c8, + 0x1c9, 0x1c9, 0x1c9, 0x1c9, 0x1ca, 0x1ca, 0x1ca, 0x1ca, 0x1cb, 0x1cb, 0x1cb, 0x1cb, 0x1cc, 0x1cc, 0x1cc, 0x1cd, + 0x1cd, 0x1cd, 0x1cd, 0x1ce, 0x1ce, 0x1ce, 0x1ce, 0x1cf, 0x1cf, 0x1cf, 0x1cf, 0x1d0, 0x1d0, 0x1d0, 0x1d1, 0x1d1, + 0x1d1, 0x1d1, 0x1d2, 0x1d2, 0x1d2, 0x1d2, 0x1d3, 0x1d3, 0x1d3, 0x1d3, 0x1d4, 0x1d4, 0x1d4, 0x1d4, 0x1d5, 0x1d5, + 0x1d5, 0x1d5, 0x1d6, 0x1d6, 0x1d6, 0x1d7, 0x1d7, 0x1d7, 0x1d7, 0x1d8, 0x1d8, 0x1d8, 0x1d8, 0x1d9, 0x1d9, 0x1d9, + 0x1d9, 0x1da, 0x1da, 0x1da, 0x1da, 0x1db, 0x1db, 0x1db, 0x1dc, 0x1dc, 0x1dc, 0x1dc, 0x1dd, 0x1dd, 0x1dd, 0x1dd, + 0x1de, 0x1de, 0x1de, 0x1de, 0x1df, 0x1df, 0x1df, 0x1df, 0x1e0, 0x1e0, 0x1e0, 0x1e0, 0x1e1, 0x1e1, 0x1e1, 0x1e2, + 0x1e2, 0x1e2, 0x1e2, 0x1e3, 0x1e3, 0x1e3, 0x1e3, 0x1e4, 0x1e4, 0x1e4, 0x1e4, 0x1e5, 0x1e5, 0x1e5, 0x1e5, 0x1e6, + 0x1e6, 0x1e6, 0x1e6, 0x1e7, 0x1e7, 0x1e7, 0x1e7, 0x1e8, 0x1e8, 0x1e8, 0x1e9, 0x1e9, 0x1e9, 0x1e9, 0x1ea, 0x1ea, + 0x1ea, 0x1ea, 0x1eb, 0x1eb, 0x1eb, 0x1eb, 0x1ec, 0x1ec, 0x1ec, 0x1ec, 0x1ed, 0x1ed, 0x1ed, 0x1ed, 0x1ee, 0x1ee, + 0x1ee, 0x1ee, 0x1ef, 0x1ef, 0x1ef, 0x1f0, 0x1f0, 0x1f0, 0x1f0, 0x1f1, 0x1f1, 0x1f1, 0x1f1, 0x1f2, 0x1f2, 0x1f2, + 0x1f2, 0x1f3, 0x1f3, 0x1f3, 0x1f3, 0x1f4, 0x1f4, 0x1f4, 0x1f4, 0x1f5, 0x1f5, 0x1f5, 0x1f5, 0x1f6, 0x1f6, 0x1f6, + 0x1f6, 0x1f7, 0x1f7, 0x1f7, 0x1f7, 0x1f8, 0x1f8, 0x1f8, 0x1f9, 0x1f9, 0x1f9, 0x1f9, 0x1fa, 0x1fa, 0x1fa, 0x1fa, + 0x1fb, 0x1fb, 0x1fb, 0x1fb, 0x1fc, 0x1fc, 0x1fc, 0x1fc, 0x1fd, 0x1fd, 0x1fd, 0x1fd, 0x1fe, 0x1fe, 0x1fe, 0x1fe, + 0x1ff, 0x1ff, 0x1ff, 0x1ff, 0x200, 0x200, 0x200, 0x200, 0x201, 0x201, 0x201, 0x201, 0x202, 0x202, 0x202, 0x202, + 0x203, 0x203, 0x203, 0x204, 0x204, 0x204, 0x204, 0x205, 0x205, 0x205, 0x205, 0x206, 0x206, 0x206, 0x206, 0x207, + 0x207, 0x207, 0x207, 0x208, 0x208, 0x208, 0x208, 0x209, 0x209, 0x209, 0x209, 0x20a, 0x20a, 0x20a, 0x20a, 0x20b, + 0x20b, 0x20b, 0x20b, 0x20c, 0x20c, 0x20c, 0x20c, 0x20d, 0x20d, 0x20d, 0x20d, 0x20e, 0x20e, 0x20e, 0x20e, 0x20f, + 0x20f, 0x20f, 0x20f, 0x210, 0x210, 0x210, 0x210, 0x211, 0x211, 0x211, 0x211, 0x212, 0x212, 0x212, 0x212, 0x213, + 0x213, 0x213, 0x213, 0x214, 0x214, 0x214, 0x214, 0x215, 0x215, 0x215, 0x215, 0x216, 0x216, 0x216, 0x216, 0x217, + 0x217, 0x217, 0x217, 0x218, 0x218, 0x218, 0x218, 0x219, 0x219, 0x219, 0x219, 0x21a, 0x21a, 0x21a, 0x21a, 0x21b, + 0x21b, 0x21b, 0x21b, 0x21c, 0x21c, 0x21c, 0x21c, 0x21d, 0x21d, 0x21d, 0x21d, 0x21e, 0x21e, 0x21e, 0x21e, 0x21f, + 0x21f, 0x21f, 0x21f, 0x220, 0x220, 0x220, 0x220, 0x221, 0x221, 0x221, 0x221, 0x222, 0x222, 0x222, 0x222, 0x223, + 0x223, 0x223, 0x223, 0x224, 0x224, 0x224, 0x224, 0x225, 0x225, 0x225, 0x225, 0x226, 0x226, 0x226, 0x226, 0x227, + 0x227, 0x227, 0x227, 0x228, 0x228, 0x228, 0x228, 0x229, 0x229, 0x229, 0x229, 0x22a, 0x22a, 0x22a, 0x22a, 0x22b, + 0x22b, 0x22b, 0x22b, 0x22c, 0x22c, 0x22c, 0x22c, 0x22d, 0x22d, 0x22d, 0x22d, 0x22e, 0x22e, 0x22e, 0x22e, 0x22f, + 0x22f, 0x22f, 0x22f, 0x230, 0x230, 0x230, 0x230, 0x231, 0x231, 0x231, 0x231, 0x232, 0x232, 0x232, 0x232, 0x233, + 0x233, 0x233, 0x233, 0x234, 0x234, 0x234, 0x234, 0x235, 0x235, 0x235, 0x235, 0x236, 0x236, 0x236, 0x236, 0x237, + 0x237, 0x237, 0x237, 0x237, 0x238, 0x238, 0x238, 0x238, 0x239, 0x239, 0x239, 0x239, 0x23a, 0x23a, 0x23a, 0x23a, + 0x23b, 0x23b, 0x23b, 0x23b, 0x23c, 0x23c, 0x23c, 0x23c, 0x23d, 0x23d, 0x23d, 0x23d, 0x23e, 0x23e, 0x23e, 0x23e, + 0x23f, 0x23f, 0x23f, 0x23f, 0x240, 0x240, 0x240, 0x240, 0x241, 0x241, 0x241, 0x241, 0x242, 0x242, 0x242, 0x242, + 0x243, 0x243, 0x243, 0x243, 0x243, 0x244, 0x244, 0x244, 0x244, 0x245, 0x245, 0x245, 0x245, 0x246, 0x246, 0x246, + 0x246, 0x247, 0x247, 0x247, 0x247, 0x248, 0x248, 0x248, 0x248, 0x249, 0x249, 0x249, 0x249, 0x24a, 0x24a, 0x24a, + 0x24a, 0x24b, 0x24b, 0x24b, 0x24b, 0x24c, 0x24c, 0x24c, 0x24c, 0x24c, 0x24d, 0x24d, 0x24d, 0x24d, 0x24e, 0x24e, + 0x24e, 0x24e, 0x24f, 0x24f, 0x24f, 0x24f, 0x250, 0x250, 0x250, 0x250, 0x251, 0x251, 0x251, 0x251, 0x252, 0x252, + 0x252, 0x252, 0x253, 0x253, 0x253, 0x253, 0x253, 0x254, 0x254, 0x254, 0x254, 0x255, 0x255, 0x255, 0x255, 0x256, + 0x256, 0x256, 0x256, 0x257, 0x257, 0x257, 0x257, 0x258, 0x258, 0x258, 0x258, 0x259, 0x259, 0x259, 0x259, 0x25a, + 0x25a, 0x25a, 0x25a, 0x25a, 0x25b, 0x25b, 0x25b, 0x25b, 0x25c, 0x25c, 0x25c, 0x25c, 0x25d, 0x25d, 0x25d, 0x25d, + 0x25e, 0x25e, 0x25e, 0x25e, 0x25f, 0x25f, 0x25f, 0x25f, 0x25f, 0x260, 0x260, 0x260, 0x260, 0x261, 0x261, 0x261, + 0x261, 0x262, 0x262, 0x262, 0x262, 0x263, 0x263, 0x263, 0x263, 0x264, 0x264, 0x264, 0x264, 0x264, 0x265, 0x265, + 0x265, 0x265, 0x266, 0x266, 0x266, 0x266, 0x267, 0x267, 0x267, 0x267, 0x268, 0x268, 0x268, 0x268, 0x269, 0x269, + 0x269, 0x269, 0x269, 0x26a, 0x26a, 0x26a, 0x26a, 0x26b, 0x26b, 0x26b, 0x26b, 0x26c, 0x26c, 0x26c, 0x26c, 0x26d, + 0x26d, 0x26d, 0x26d, 0x26e, 0x26e, 0x26e, 0x26e, 0x26e, 0x26f, 0x26f, 0x26f, 0x26f, 0x270, 0x270, 0x270, 0x270, + 0x271, 0x271, 0x271, 0x271, 0x272, 0x272, 0x272, 0x272, 0x272, 0x273, 0x273, 0x273, 0x273, 0x274, 0x274, 0x274, + 0x274, 0x275, 0x275, 0x275, 0x275, 0x276, 0x276, 0x276, 0x276, 0x276, 0x277, 0x277, 0x277, 0x277, 0x278, 0x278, + 0x278, 0x278, 0x279, 0x279, 0x279, 0x279, 0x27a, 0x27a, 0x27a, 0x27a, 0x27a, 0x27b, 0x27b, 0x27b, 0x27b, 0x27c, + 0x27c, 0x27c, 0x27c, 0x27d, 0x27d, 0x27d, 0x27d, 0x27e, 0x27e, 0x27e, 0x27e, 0x27e, 0x27f, 0x27f, 0x27f, 0x27f, + 0x280, 0x280, 0x280, 0x280, 0x281, 0x281, 0x281, 0x281, 0x282, 0x282, 0x282, 0x282, 0x282, 0x283, 0x283, 0x283, + 0x283, 0x284, 0x284, 0x284, 0x284, 0x285, 0x285, 0x285, 0x285, 0x285, 0x286, 0x286, 0x286, 0x286, 0x287, 0x287, + 0x287, 0x287, 0x288, 0x288, 0x288, 0x288, 0x289, 0x289, 0x289, 0x289, 0x289, 0x28a, 0x28a, 0x28a, 0x28a, 0x28b, + 0x28b, 0x28b, 0x28b, 0x28c, 0x28c, 0x28c, 0x28c, 0x28c, 0x28d, 0x28d, 0x28d, 0x28d, 0x28e, 0x28e, 0x28e, 0x28e, + 0x28f, 0x28f, 0x28f, 0x28f, 0x28f, 0x290, 0x290, 0x290, 0x290, 0x291, 0x291, 0x291, 0x291, 0x292, 0x292, 0x292, + 0x292, 0x292, 0x293, 0x293, 0x293, 0x293, 0x294, 0x294, 0x294, 0x294, 0x295, 0x295, 0x295, 0x295, 0x295, 0x296, + 0x296, 0x296, 0x296, 0x297, 0x297, 0x297, 0x297, 0x298, 0x298, 0x298, 0x298, 0x298, 0x299, 0x299, 0x299, 0x299, + 0x29a, 0x29a, 0x29a, 0x29a, 0x29b, 0x29b, 0x29b, 0x29b, 0x29b, 0x29c, 0x29c, 0x29c, 0x29c, 0x29d, 0x29d, 0x29d, + 0x29d, 0x29e, 0x29e, 0x29e, 0x29e, 0x29e, 0x29f, 0x29f, 0x29f, 0x29f, 0x2a0, 0x2a0, 0x2a0, 0x2a0, 0x2a1, 0x2a1, + 0x2a1, 0x2a1, 0x2a1, 0x2a2, 0x2a2, 0x2a2, 0x2a2, 0x2a3, 0x2a3, 0x2a3, 0x2a3, 0x2a3, 0x2a4, 0x2a4, 0x2a4, 0x2a4, + 0x2a5, 0x2a5, 0x2a5, 0x2a5, 0x2a6, 0x2a6, 0x2a6, 0x2a6, 0x2a6, 0x2a7, 0x2a7, 0x2a7, 0x2a7, 0x2a8, 0x2a8, 0x2a8, + 0x2a8, 0x2a8, 0x2a9, 0x2a9, 0x2a9, 0x2a9, 0x2aa, 0x2aa, 0x2aa, 0x2aa, 0x2ab, 0x2ab, 0x2ab, 0x2ab, 0x2ab, 0x2ac, + 0x2ac, 0x2ac, 0x2ac, 0x2ad, 0x2ad, 0x2ad, 0x2ad, 0x2ad, 0x2ae, 0x2ae, 0x2ae, 0x2ae, 0x2af, 0x2af, 0x2af, 0x2af, + 0x2b0, 0x2b0, 0x2b0, 0x2b0, 0x2b0, 0x2b1, 0x2b1, 0x2b1, 0x2b1, 0x2b2, 0x2b2, 0x2b2, 0x2b2, 0x2b2, 0x2b3, 0x2b3, + 0x2b3, 0x2b3, 0x2b4, 0x2b4, 0x2b4, 0x2b4, 0x2b5, 0x2b5, 0x2b5, 0x2b5, 0x2b5, 0x2b6, 0x2b6, 0x2b6, 0x2b6, 0x2b7, + 0x2b7, 0x2b7, 0x2b7, 0x2b7, 0x2b8, 0x2b8, 0x2b8, 0x2b8, 0x2b9, 0x2b9, 0x2b9, 0x2b9, 0x2b9, 0x2ba, 0x2ba, 0x2ba, + 0x2ba, 0x2bb, 0x2bb, 0x2bb, 0x2bb, 0x2bb, 0x2bc, 0x2bc, 0x2bc, 0x2bc, 0x2bd, 0x2bd, 0x2bd, 0x2bd, 0x2be, 0x2be, + 0x2be, 0x2be, 0x2be, 0x2bf, 0x2bf, 0x2bf, 0x2bf, 0x2c0, 0x2c0, 0x2c0, 0x2c0, 0x2c0, 0x2c1, 0x2c1, 0x2c1, 0x2c1, + 0x2c2, 0x2c2, 0x2c2, 0x2c2, 0x2c2, 0x2c3, 0x2c3, 0x2c3, 0x2c3, 0x2c4, 0x2c4, 0x2c4, 0x2c4, 0x2c4, 0x2c5, 0x2c5, + 0x2c5, 0x2c5, 0x2c6, 0x2c6, 0x2c6, 0x2c6, 0x2c6, 0x2c7, 0x2c7, 0x2c7, 0x2c7, 0x2c8, 0x2c8, 0x2c8, 0x2c8, 0x2c8, + 0x2c9, 0x2c9, 0x2c9, 0x2c9, 0x2ca, 0x2ca, 0x2ca, 0x2ca, 0x2ca, 0x2cb, 0x2cb, 0x2cb, 0x2cb, 0x2cc, 0x2cc, 0x2cc, + 0x2cc, 0x2cc, 0x2cd, 0x2cd, 0x2cd, 0x2cd, 0x2ce, 0x2ce, 0x2ce, 0x2ce, 0x2ce, 0x2cf, 0x2cf, 0x2cf, 0x2cf, 0x2d0, + 0x2d0, 0x2d0, 0x2d0, 0x2d0, 0x2d1, 0x2d1, 0x2d1, 0x2d1, 0x2d2, 0x2d2, 0x2d2, 0x2d2, 0x2d2, 0x2d3, 0x2d3, 0x2d3, + 0x2d3, 0x2d4, 0x2d4, 0x2d4, 0x2d4, 0x2d4, 0x2d5, 0x2d5, 0x2d5, 0x2d5, 0x2d6, 0x2d6, 0x2d6, 0x2d6, 0x2d6, 0x2d7, + 0x2d7, 0x2d7, 0x2d7, 0x2d8, 0x2d8, 0x2d8, 0x2d8, 0x2d8, 0x2d9, 0x2d9, 0x2d9, 0x2d9, 0x2da, 0x2da, 0x2da, 0x2da, + 0x2da, 0x2db, 0x2db, 0x2db, 0x2db, 0x2db, 0x2dc, 0x2dc, 0x2dc, 0x2dc, 0x2dd, 0x2dd, 0x2dd, 0x2dd, 0x2dd, 0x2de, + 0x2de, 0x2de, 0x2de, 0x2df, 0x2df, 0x2df, 0x2df, 0x2df, 0x2e0, 0x2e0, 0x2e0, 0x2e0, 0x2e1, 0x2e1, 0x2e1, 0x2e1, + 0x2e1, 0x2e2, 0x2e2, 0x2e2, 0x2e2, 0x2e3, 0x2e3, 0x2e3, 0x2e3, 0x2e3, 0x2e4, 0x2e4, 0x2e4, 0x2e4, 0x2e4, 0x2e5, + 0x2e5, 0x2e5, 0x2e5, 0x2e6, 0x2e6, 0x2e6, 0x2e6, 0x2e6, 0x2e7, 0x2e7, 0x2e7, 0x2e7, 0x2e8, 0x2e8, 0x2e8, 0x2e8, + 0x2e8, 0x2e9, 0x2e9, 0x2e9, 0x2e9, 0x2ea, 0x2ea, 0x2ea, 0x2ea, 0x2ea, 0x2eb, 0x2eb, 0x2eb, 0x2eb, 0x2eb, 0x2ec, + 0x2ec, 0x2ec, 0x2ec, 0x2ed, 0x2ed, 0x2ed, 0x2ed, 0x2ed, 0x2ee, 0x2ee, 0x2ee, 0x2ee, 0x2ef, 0x2ef, 0x2ef, 0x2ef, + 0x2ef, 0x2f0, 0x2f0, 0x2f0, 0x2f0, 0x2f0, 0x2f1, 0x2f1, 0x2f1, 0x2f1, 0x2f2, 0x2f2, 0x2f2, 0x2f2, 0x2f2, 0x2f3, + 0x2f3, 0x2f3, 0x2f3, 0x2f3, 0x2f4, 0x2f4, 0x2f4, 0x2f4, 0x2f5, 0x2f5, 0x2f5, 0x2f5, 0x2f5, 0x2f6, 0x2f6, 0x2f6, + 0x2f6, 0x2f7, 0x2f7, 0x2f7, 0x2f7, 0x2f7, 0x2f8, 0x2f8, 0x2f8, 0x2f8, 0x2f8, 0x2f9, 0x2f9, 0x2f9, 0x2f9, 0x2fa, + 0x2fa, 0x2fa, 0x2fa, 0x2fa, 0x2fb, 0x2fb, 0x2fb, 0x2fb, 0x2fb, 0x2fc, 0x2fc, 0x2fc, 0x2fc, 0x2fd, 0x2fd, 0x2fd, + 0x2fd, 0x2fd, 0x2fe, 0x2fe, 0x2fe, 0x2fe, 0x2fe, 0x2ff, 0x2ff, 0x2ff, 0x2ff, 0x300, 0x300, 0x300, 0x300, 0x300, + 0x301, 0x301, 0x301, 0x301, 0x301, 0x302, 0x302, 0x302, 0x302, 0x303, 0x303, 0x303, 0x303, 0x303, 0x304, 0x304, + 0x304, 0x304, 0x304, 0x305, 0x305, 0x305, 0x305, 0x306, 0x306, 0x306, 0x306, 0x306, 0x307, 0x307, 0x307, 0x307, + 0x307, 0x308, 0x308, 0x308, 0x308, 0x309, 0x309, 0x309, 0x309, 0x309, 0x30a, 0x30a, 0x30a, 0x30a, 0x30a, 0x30b, + 0x30b, 0x30b, 0x30b, 0x30c, 0x30c, 0x30c, 0x30c, 0x30c, 0x30d, 0x30d, 0x30d, 0x30d, 0x30d, 0x30e, 0x30e, 0x30e, + 0x30e, 0x30e, 0x30f, 0x30f, 0x30f, 0x30f, 0x310, 0x310, 0x310, 0x310, 0x310, 0x311, 0x311, 0x311, 0x311, 0x311, + 0x312, 0x312, 0x312, 0x312, 0x313, 0x313, 0x313, 0x313, 0x313, 0x314, 0x314, 0x314, 0x314, 0x314, 0x315, 0x315, + 0x315, 0x315, 0x315, 0x316, 0x316, 0x316, 0x316, 0x317, 0x317, 0x317, 0x317, 0x317, 0x318, 0x318, 0x318, 0x318, + 0x318, 0x319, 0x319, 0x319, 0x319, 0x319, 0x31a, 0x31a, 0x31a, 0x31a, 0x31b, 0x31b, 0x31b, 0x31b, 0x31b, 0x31c, + 0x31c, 0x31c, 0x31c, 0x31c, 0x31d, 0x31d, 0x31d, 0x31d, 0x31e, 0x31e, 0x31e, 0x31e, 0x31e, 0x31f, 0x31f, 0x31f, + 0x31f, 0x31f, 0x320, 0x320, 0x320, 0x320, 0x320, 0x321, 0x321, 0x321, 0x321, 0x321, 0x322, 0x322, 0x322, 0x322, + 0x323, 0x323, 0x323, 0x323, 0x323, 0x324, 0x324, 0x324, 0x324, 0x324, 0x325, 0x325, 0x325, 0x325, 0x325, 0x326, + 0x326, 0x326, 0x326, 0x327, 0x327, 0x327, 0x327, 0x327, 0x328, 0x328, 0x328, 0x328, 0x328, 0x329, 0x329, 0x329, + 0x329, 0x329, 0x32a, 0x32a, 0x32a, 0x32a, 0x32a, 0x32b, 0x32b, 0x32b, 0x32b, 0x32c, 0x32c, 0x32c, 0x32c, 0x32c, + 0x32d, 0x32d, 0x32d, 0x32d, 0x32d, 0x32e, 0x32e, 0x32e, 0x32e, 0x32e, 0x32f, 0x32f, 0x32f, 0x32f, 0x32f, 0x330, + 0x330, 0x330, 0x330, 0x331, 0x331, 0x331, 0x331, 0x331, 0x332, 0x332, 0x332, 0x332, 0x332, 0x333, 0x333, 0x333, + 0x333, 0x333, 0x334, 0x334, 0x334, 0x334, 0x334, 0x335, 0x335, 0x335, 0x335, 0x335, 0x336, 0x336, 0x336, 0x336, + 0x337, 0x337, 0x337, 0x337, 0x337, 0x338, 0x338, 0x338, 0x338, 0x338, 0x339, 0x339, 0x339, 0x339, 0x339, 0x33a, + 0x33a, 0x33a, 0x33a, 0x33a, 0x33b, 0x33b, 0x33b, 0x33b, 0x33b, 0x33c, 0x33c, 0x33c, 0x33c, 0x33d, 0x33d, 0x33d, + 0x33d, 0x33d, 0x33e, 0x33e, 0x33e, 0x33e, 0x33e, 0x33f, 0x33f, 0x33f, 0x33f, 0x33f, 0x340, 0x340, 0x340, 0x340, + 0x340, 0x341, 0x341, 0x341, 0x341, 0x341, 0x342, 0x342, 0x342, 0x342, 0x342, 0x343, 0x343, 0x343, 0x343, 0x344, + 0x344, 0x344, 0x344, 0x344, 0x345, 0x345, 0x345, 0x345, 0x345, 0x346, 0x346, 0x346, 0x346, 0x346, 0x347, 0x347, + 0x347, 0x347, 0x347, 0x348, 0x348, 0x348, 0x348, 0x348, 0x349, 0x349, 0x349, 0x349, 0x349, 0x34a, 0x34a, 0x34a, + 0x34a, 0x34a, 0x34b, 0x34b, 0x34b, 0x34b, 0x34b, 0x34c, 0x34c, 0x34c, 0x34c, 0x34c, 0x34d, 0x34d, 0x34d, 0x34d, + 0x34d, 0x34e, 0x34e, 0x34e, 0x34e, 0x34f, 0x34f, 0x34f, 0x34f, 0x34f, 0x350, 0x350, 0x350, 0x350, 0x350, 0x351, + 0x351, 0x351, 0x351, 0x351, 0x352, 0x352, 0x352, 0x352, 0x352, 0x353, 0x353, 0x353, 0x353, 0x353, 0x354, 0x354, + 0x354, 0x354, 0x354, 0x355, 0x355, 0x355, 0x355, 0x355, 0x356, 0x356, 0x356, 0x356, 0x356, 0x357, 0x357, 0x357, + 0x357, 0x357, 0x358, 0x358, 0x358, 0x358, 0x358, 0x359, 0x359, 0x359, 0x359, 0x359, 0x35a, 0x35a, 0x35a, 0x35a, + 0x35a, 0x35b, 0x35b, 0x35b, 0x35b, 0x35b, 0x35c, 0x35c, 0x35c, 0x35c, 0x35c, 0x35d, 0x35d, 0x35d, 0x35d, 0x35d, + 0x35e, 0x35e, 0x35e, 0x35e, 0x35e, 0x35f, 0x35f, 0x35f, 0x35f, 0x35f, 0x360, 0x360, 0x360, 0x360, 0x360, 0x361, + 0x361, 0x361, 0x361, 0x362, 0x362, 0x362, 0x362, 0x362, 0x363, 0x363, 0x363, 0x363, 0x363, 0x364, 0x364, 0x364, + 0x364, 0x364, 0x365, 0x365, 0x365, 0x365, 0x365, 0x366, 0x366, 0x366, 0x366, 0x366, 0x367, 0x367, 0x367, 0x367, + 0x367, 0x368, 0x368, 0x368, 0x368, 0x368, 0x369, 0x369, 0x369, 0x369, 0x369, 0x36a, 0x36a, 0x36a, 0x36a, 0x36a, + 0x36b, 0x36b, 0x36b, 0x36b, 0x36b, 0x36c, 0x36c, 0x36c, 0x36c, 0x36c, 0x36d, 0x36d, 0x36d, 0x36d, 0x36d, 0x36e, + 0x36e, 0x36e, 0x36e, 0x36e, 0x36f, 0x36f, 0x36f, 0x36f, 0x36f, 0x370, 0x370, 0x370, 0x370, 0x370, 0x371, 0x371, + 0x371, 0x371, 0x371, 0x371, 0x372, 0x372, 0x372, 0x372, 0x372, 0x373, 0x373, 0x373, 0x373, 0x373, 0x374, 0x374, + 0x374, 0x374, 0x374, 0x375, 0x375, 0x375, 0x375, 0x375, 0x376, 0x376, 0x376, 0x376, 0x376, 0x377, 0x377, 0x377, + 0x377, 0x377, 0x378, 0x378, 0x378, 0x378, 0x378, 0x379, 0x379, 0x379, 0x379, 0x379, 0x37a, 0x37a, 0x37a, 0x37a, + 0x37a, 0x37b, 0x37b, 0x37b, 0x37b, 0x37b, 0x37c, 0x37c, 0x37c, 0x37c, 0x37c, 0x37d, 0x37d, 0x37d, 0x37d, 0x37d, + 0x37e, 0x37e, 0x37e, 0x37e, 0x37e, 0x37f, 0x37f, 0x37f, 0x37f, 0x37f, 0x380, 0x380, 0x380, 0x380, 0x380, 0x381, + 0x381, 0x381, 0x381, 0x381, 0x381, 0x382, 0x382, 0x382, 0x382, 0x382, 0x383, 0x383, 0x383, 0x383, 0x383, 0x384, + 0x384, 0x384, 0x384, 0x384, 0x385, 0x385, 0x385, 0x385, 0x385, 0x386, 0x386, 0x386, 0x386, 0x386, 0x387, 0x387, + 0x387, 0x387, 0x387, 0x388, 0x388, 0x388, 0x388, 0x388, 0x389, 0x389, 0x389, 0x389, 0x389, 0x38a, 0x38a, 0x38a, + 0x38a, 0x38a, 0x38a, 0x38b, 0x38b, 0x38b, 0x38b, 0x38b, 0x38c, 0x38c, 0x38c, 0x38c, 0x38c, 0x38d, 0x38d, 0x38d, + 0x38d, 0x38d, 0x38e, 0x38e, 0x38e, 0x38e, 0x38e, 0x38f, 0x38f, 0x38f, 0x38f, 0x38f, 0x390, 0x390, 0x390, 0x390, + 0x390, 0x391, 0x391, 0x391, 0x391, 0x391, 0x392, 0x392, 0x392, 0x392, 0x392, 0x392, 0x393, 0x393, 0x393, 0x393, + 0x393, 0x394, 0x394, 0x394, 0x394, 0x394, 0x395, 0x395, 0x395, 0x395, 0x395, 0x396, 0x396, 0x396, 0x396, 0x396, + 0x397, 0x397, 0x397, 0x397, 0x397, 0x398, 0x398, 0x398, 0x398, 0x398, 0x399, 0x399, 0x399, 0x399, 0x399, 0x399, + 0x39a, 0x39a, 0x39a, 0x39a, 0x39a, 0x39b, 0x39b, 0x39b, 0x39b, 0x39b, 0x39c, 0x39c, 0x39c, 0x39c, 0x39c, 0x39d, + 0x39d, 0x39d, 0x39d, 0x39d, 0x39e, 0x39e, 0x39e, 0x39e, 0x39e, 0x39e, 0x39f, 0x39f, 0x39f, 0x39f, 0x39f, 0x3a0, + 0x3a0, 0x3a0, 0x3a0, 0x3a0, 0x3a1, 0x3a1, 0x3a1, 0x3a1, 0x3a1, 0x3a2, 0x3a2, 0x3a2, 0x3a2, 0x3a2, 0x3a3, 0x3a3, + 0x3a3, 0x3a3, 0x3a3, 0x3a3, 0x3a4, 0x3a4, 0x3a4, 0x3a4, 0x3a4, 0x3a5, 0x3a5, 0x3a5, 0x3a5, 0x3a5, 0x3a6, 0x3a6, + 0x3a6, 0x3a6, 0x3a6, 0x3a7, 0x3a7, 0x3a7, 0x3a7, 0x3a7, 0x3a8, 0x3a8, 0x3a8, 0x3a8, 0x3a8, 0x3a8, 0x3a9, 0x3a9, + 0x3a9, 0x3a9, 0x3a9, 0x3aa, 0x3aa, 0x3aa, 0x3aa, 0x3aa, 0x3ab, 0x3ab, 0x3ab, 0x3ab, 0x3ab, 0x3ac, 0x3ac, 0x3ac, + 0x3ac, 0x3ac, 0x3ac, 0x3ad, 0x3ad, 0x3ad, 0x3ad, 0x3ad, 0x3ae, 0x3ae, 0x3ae, 0x3ae, 0x3ae, 0x3af, 0x3af, 0x3af, + 0x3af, 0x3af, 0x3b0, 0x3b0, 0x3b0, 0x3b0, 0x3b0, 0x3b0, 0x3b1, 0x3b1, 0x3b1, 0x3b1, 0x3b1, 0x3b2, 0x3b2, 0x3b2, + 0x3b2, 0x3b2, 0x3b3, 0x3b3, 0x3b3, 0x3b3, 0x3b3, 0x3b4, 0x3b4, 0x3b4, 0x3b4, 0x3b4, 0x3b4, 0x3b5, 0x3b5, 0x3b5, + 0x3b5, 0x3b5, 0x3b6, 0x3b6, 0x3b6, 0x3b6, 0x3b6, 0x3b7, 0x3b7, 0x3b7, 0x3b7, 0x3b7, 0x3b8, 0x3b8, 0x3b8, 0x3b8, + 0x3b8, 0x3b8, 0x3b9, 0x3b9, 0x3b9, 0x3b9, 0x3b9, 0x3ba, 0x3ba, 0x3ba, 0x3ba, 0x3ba, 0x3bb, 0x3bb, 0x3bb, 0x3bb, + 0x3bb, 0x3bb, 0x3bc, 0x3bc, 0x3bc, 0x3bc, 0x3bc, 0x3bd, 0x3bd, 0x3bd, 0x3bd, 0x3bd, 0x3be, 0x3be, 0x3be, 0x3be, + 0x3be, 0x3bf, 0x3bf, 0x3bf, 0x3bf, 0x3bf, 0x3bf, 0x3c0, 0x3c0, 0x3c0, 0x3c0, 0x3c0, 0x3c1, 0x3c1, 0x3c1, 0x3c1, + 0x3c1, 0x3c2, 0x3c2, 0x3c2, 0x3c2, 0x3c2, 0x3c2, 0x3c3, 0x3c3, 0x3c3, 0x3c3, 0x3c3, 0x3c4, 0x3c4, 0x3c4, 0x3c4, + 0x3c4, 0x3c5, 0x3c5, 0x3c5, 0x3c5, 0x3c5, 0x3c5, 0x3c6, 0x3c6, 0x3c6, 0x3c6, 0x3c6, 0x3c7, 0x3c7, 0x3c7, 0x3c7, + 0x3c7, 0x3c8, 0x3c8, 0x3c8, 0x3c8, 0x3c8, 0x3c8, 0x3c9, 0x3c9, 0x3c9, 0x3c9, 0x3c9, 0x3ca, 0x3ca, 0x3ca, 0x3ca, + 0x3ca, 0x3cb, 0x3cb, 0x3cb, 0x3cb, 0x3cb, 0x3cb, 0x3cc, 0x3cc, 0x3cc, 0x3cc, 0x3cc, 0x3cd, 0x3cd, 0x3cd, 0x3cd, + 0x3cd, 0x3ce, 0x3ce, 0x3ce, 0x3ce, 0x3ce, 0x3ce, 0x3cf, 0x3cf, 0x3cf, 0x3cf, 0x3cf, 0x3d0, 0x3d0, 0x3d0, 0x3d0, + 0x3d0, 0x3d0, 0x3d1, 0x3d1, 0x3d1, 0x3d1, 0x3d1, 0x3d2, 0x3d2, 0x3d2, 0x3d2, 0x3d2, 0x3d3, 0x3d3, 0x3d3, 0x3d3, + 0x3d3, 0x3d3, 0x3d4, 0x3d4, 0x3d4, 0x3d4, 0x3d4, 0x3d5, 0x3d5, 0x3d5, 0x3d5, 0x3d5, 0x3d6, 0x3d6, 0x3d6, 0x3d6, + 0x3d6, 0x3d6, 0x3d7, 0x3d7, 0x3d7, 0x3d7, 0x3d7, 0x3d8, 0x3d8, 0x3d8, 0x3d8, 0x3d8, 0x3d8, 0x3d9, 0x3d9, 0x3d9, + 0x3d9, 0x3d9, 0x3da, 0x3da, 0x3da, 0x3da, 0x3da, 0x3db, 0x3db, 0x3db, 0x3db, 0x3db, 0x3db, 0x3dc, 0x3dc, 0x3dc, + 0x3dc, 0x3dc, 0x3dd, 0x3dd, 0x3dd, 0x3dd, 0x3dd, 0x3dd, 0x3de, 0x3de, 0x3de, 0x3de, 0x3de, 0x3df, 0x3df, 0x3df, + 0x3df, 0x3df, 0x3df, 0x3e0, 0x3e0, 0x3e0, 0x3e0, 0x3e0, 0x3e1, 0x3e1, 0x3e1, 0x3e1, 0x3e1, 0x3e2, 0x3e2, 0x3e2, + 0x3e2, 0x3e2, 0x3e2, 0x3e3, 0x3e3, 0x3e3, 0x3e3, 0x3e3, 0x3e4, 0x3e4, 0x3e4, 0x3e4, 0x3e4, 0x3e4, 0x3e5, 0x3e5, + 0x3e5, 0x3e5, 0x3e5, 0x3e6, 0x3e6, 0x3e6, 0x3e6, 0x3e6, 0x3e6, 0x3e7, 0x3e7, 0x3e7, 0x3e7, 0x3e7, 0x3e8, 0x3e8, + 0x3e8, 0x3e8, 0x3e8, 0x3e9, 0x3e9, 0x3e9, 0x3e9, 0x3e9, 0x3e9, 0x3ea, 0x3ea, 0x3ea, 0x3ea, 0x3ea, 0x3eb, 0x3eb, + 0x3eb, 0x3eb, 0x3eb, 0x3eb, 0x3ec, 0x3ec, 0x3ec, 0x3ec, 0x3ec, 0x3ed, 0x3ed, 0x3ed, 0x3ed, 0x3ed, 0x3ed, 0x3ee, + 0x3ee, 0x3ee, 0x3ee, 0x3ee, 0x3ef, 0x3ef, 0x3ef, 0x3ef, 0x3ef, 0x3ef, 0x3f0, 0x3f0, 0x3f0, 0x3f0, 0x3f0, 0x3f1, + 0x3f1, 0x3f1, 0x3f1, 0x3f1, 0x3f1, 0x3f2, 0x3f2, 0x3f2, 0x3f2, 0x3f2, 0x3f3, 0x3f3, 0x3f3, 0x3f3, 0x3f3, 0x3f3, + 0x3f4, 0x3f4, 0x3f4, 0x3f4, 0x3f4, 0x3f5, 0x3f5, 0x3f5, 0x3f5, 0x3f5, 0x3f5, 0x3f6, 0x3f6, 0x3f6, 0x3f6, 0x3f6, + 0x3f7, 0x3f7, 0x3f7, 0x3f7, 0x3f7, 0x3f7, 0x3f8, 0x3f8, 0x3f8, 0x3f8, 0x3f8, 0x3f9, 0x3f9, 0x3f9, 0x3f9, 0x3f9, + 0x3f9, 0x3fa, 0x3fa, 0x3fa, 0x3fa, 0x3fa, 0x3fb, 0x3fb, 0x3fb, 0x3fb, 0x3fb, 0x3fb, 0x3fc, 0x3fc, 0x3fc, 0x3fc, + 0x3fc, 0x3fd, 0x3fd, 0x3fd, 0x3fd, 0x3fd, 0x3fd, 0x3fe, 0x3fe, 0x3fe, 0x3fe, 0x3fe, 0x3fe, 0x3ff, 0x3ff, 0x3ff, +}; + +static uint8_t chartbl[512] = { + 9, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; diff --git a/winzipjpeg/ArithmeticDecoder.h b/winzipjpeg/ArithmeticDecoder.h new file mode 100644 index 0000000..d4d25b2 --- /dev/null +++ b/winzipjpeg/ArithmeticDecoder.h @@ -0,0 +1,71 @@ +/* + * ArithmeticDecoder.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __WINZIP_JPEG_ARITHMETIC_DECODER_H__ +#define __WINZIP_JPEG_ARITHMETIC_DECODER_H__ + +#include "InputStream.h" + +#include +#include + +typedef struct WinZipJPEGArithmeticDecoder +{ + WinZipJPEGReadFunction *readfunc; + void *inputcontext; + + bool eof; + + uint8_t currbyte, lastbyte; + + uint8_t kmin2; // LPS count for reduction of Q by 4 + uint8_t kmin1; // LPS count for reduction of Q by 2 + uint8_t kmin; // largest LSP[sic] count for smaller Q + // uint8_t kavg; // expected average LPS count + uint8_t kmax; // smallest LPS count for larger Q + uint32_t x; // finite pricesion window on code stream + int32_t lp; // minus log p --- used only for testing + int32_t lr; // minus log of the range + int32_t lrm; // maximum lr before change index + int32_t lx; // decoder - log x + uint32_t dx; // antilog of lr -- used only for testing +} WinZipJPEGArithmeticDecoder; + +typedef struct WinZipJPEGContext +{ + int i; + int32_t dlrm; // difference between lrm and lr + uint8_t mps; // most probable symbol value - 0 or 1 + uint8_t k; // least probable symbol count +} WinZipJPEGContext; + +void InitializeWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGReadFunction *readfunc, + void *inputcontext); +void InitializeWinZipJPEGContext(WinZipJPEGContext *self); +void InitializeWinZipJPEGContexts(WinZipJPEGContext *first, size_t bytes); +void InitializeFixedWinZipJPEGContext(WinZipJPEGContext *self); + +int NextBitFromWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self, WinZipJPEGContext *context); + +void FlushWinZipJPEGArithmeticDecoder(WinZipJPEGArithmeticDecoder *self); + +static inline bool WinZipJPEGArithmeticDecoderEncounteredEOF(WinZipJPEGArithmeticDecoder *self) { return self->eof; } + +#endif diff --git a/winzipjpeg/Decompressor.c b/winzipjpeg/Decompressor.c new file mode 100644 index 0000000..e22b924 --- /dev/null +++ b/winzipjpeg/Decompressor.c @@ -0,0 +1,1079 @@ +/* + * Decompressor.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "Decompressor.h" +#include "LZMA.h" + +#define analyser_assert(x) ((void)0) + +#include +#include + + + +// +// Constructor and destructor. +// + +WinZipJPEGDecompressor *AllocWinZipJPEGDecompressor(WinZipJPEGReadFunction *readfunc,void *inputcontext) +{ + WinZipJPEGDecompressor *self=calloc(sizeof(WinZipJPEGDecompressor),1); + if(!self) return NULL; + + self->readfunc=readfunc; + self->inputcontext=inputcontext; + + self->metadatalength=0; + self->metadatabytes=NULL; + + self->isfirstbundle=true; + self->reachedend=false; + + self->slicesavailable=false; + + memset(self->blocks,0,sizeof(self->blocks)); + + self->mcusavailable=false; + + InitializeFixedWinZipJPEGContext(&self->fixedcontext); + + return self; +} + +void FreeWinZipJPEGDecompressor(WinZipJPEGDecompressor *self) +{ + if(!self) return; + + free(self->metadatabytes); + for(int i=0;i<4;i++) free(self->blocks[i]); + free(self); +} + + + + +// +// Header and bundle reading. +// + +// Little endian integer parsing functions. +static inline uint16_t LittleEndianUInt16(uint8_t *ptr) { return ptr[0]|(ptr[1]<<8); } +static inline uint32_t LittleEndianUInt32(uint8_t *ptr) { return ptr[0]|(ptr[1]<<8)|(ptr[2]<<16)|(ptr[3]<<24); } + +// Allocator functions for LZMA. +static void *Alloc(ISzAllocPtr p,size_t size) { (void)p; return malloc(size); } +static void Free(ISzAllocPtr p,void *address) { (void)p; return free(address); } +static ISzAlloc lzmaallocator={Alloc,Free}; + +// Helper functions for reading from the input stream. +static int FullRead(WinZipJPEGDecompressor *self,uint8_t *buffer,size_t length); +static int SkipBytes(WinZipJPEGDecompressor *self,size_t length); + +int ReadWinZipJPEGHeader(WinZipJPEGDecompressor *self) +{ + // Read 4-byte header. + uint8_t header[4]; + int error=FullRead(self,header,sizeof(header)); + if(error) return error; + + // Sanity check the header, and make sure it contains only versions we can handle. + if(header[0]<4) return WinZipJPEGInvalidHeaderError; + if(header[1]!=0x10) return WinZipJPEGInvalidHeaderError; + if(header[2]!=0x01) return WinZipJPEGInvalidHeaderError; + if(header[3]&0xe0) return WinZipJPEGInvalidHeaderError; + + // The header can possibly be bigger than 4 bytes, so skip the rest. + // (Unlikely to happen). + if(header[0]>4) + { + int error=SkipBytes(self,header[0]-4); + if(error) return error; + } + + // Parse slice value. + self->slicevalue=header[3]&0x1f; + + return WinZipJPEGNoError; +} + +int ReadNextWinZipJPEGBundle(WinZipJPEGDecompressor *self) +{ + // Free and clear any old metadata. + free(self->metadatabytes); + self->metadatalength=0; + self->metadatabytes=NULL; + + // Free and clear old slices. + for(int i=0;i<4;i++) free(self->blocks[i]); + memset(self->blocks,0,sizeof(self->blocks)); + + // Read bundle header. + uint8_t header[4]; + int error=FullRead(self,header,sizeof(header)); + if(error) return error; + + // Parse metadata sizes from header. + uint32_t uncompressedsize=LittleEndianUInt16(&header[0]); + uint32_t compressedsize=LittleEndianUInt16(&header[2]); + + // If the sizes do not fit in 16 bits, both are set to 0xffff and + // an 8-byte 32-bit header is appended. + if(uncompressedsize==0xffff && compressedsize==0xffff) + { + uint8_t header[8]; + int error=FullRead(self,header,sizeof(header)); + if(error) return error; + + uncompressedsize=LittleEndianUInt32(&header[0]); + compressedsize=LittleEndianUInt32(&header[4]); + } + + // Allocate space for the uncompressed metadata. + self->metadatabytes=malloc(uncompressedsize); + if(!self->metadatabytes) return WinZipJPEGOutOfMemoryError; + self->metadatalength=uncompressedsize; + + // NOTE: The spec does not mention this, but a compressed + // size of 0 means uncompressed data is stored. + if(compressedsize) + { + // Allocate temporary space for the compressed metadata, and read it. + uint8_t *compressedbytes=malloc(compressedsize); + if(!compressedbytes) return WinZipJPEGOutOfMemoryError; + + error=FullRead(self,compressedbytes,compressedsize); + if(error) { free(compressedbytes); return error; } + + // Calculate the dictionary size used for the LZMA coding. + int dictionarysize=(uncompressedsize+511)&~511; + if(dictionarysize<1024) dictionarysize=1024; // Silly - LZMA enforces a lower limit of 4096. + if(dictionarysize>512*1024) dictionarysize=512*1024; + + // Create properties chunk for LZMA, using the dictionary size and default settings (lc=3, lp=0, pb=2). + uint8_t properties[5]={3+0*9+2*5*9,dictionarysize,dictionarysize>>8,dictionarysize>>16,dictionarysize>>24}; + + // Run LZMA decompressor. + SizeT destlen=uncompressedsize,srclen=compressedsize; + ELzmaStatus status; + SRes res=LzmaDecode(self->metadatabytes,&destlen,compressedbytes,&srclen, + properties,sizeof(properties),LZMA_FINISH_END,&status,&lzmaallocator); + + // Free temporary buffer. + free(compressedbytes); + + // Check if LZMA decoding succeeded. + if(res!=SZ_OK) return WinZipJPEGLZMAError; + } + else + { + // Read uncompressed metadata. + error=FullRead(self,self->metadatabytes,uncompressedsize); + if(error) return error; + } + + // Parse the JPEG structure. If this is the first bundle, + // we have to first find the start marker. + const uint8_t *metadatastart; + if(self->isfirstbundle) + { + metadatastart=FindStartOfWinZipJPEGImage(self->metadatabytes,self->metadatalength); + if(!metadatastart) return WinZipJPEGParseError; + + self->isfirstbundle=false; + } + else + { + metadatastart=self->metadatabytes; + } + + int parseres=ParseWinZipJPEGMetadata(&self->jpeg,metadatastart, + self->metadatabytes+self->metadatalength-metadatastart); + if(parseres==WinZipJPEGMetadataParsingFailed) return WinZipJPEGParseError; + + // If we encountered an End Of Image marker, there will be + // no further scans or bundles, so set a flag and return. + if(parseres==WinZipJPEGMetadataFoundEndOfImage) + { + self->reachedend=true; + return WinZipJPEGNoError; + } + + // Initialize arithmetic decoder contexts. + InitializeWinZipJPEGContexts(&self->eobbins[0][0][0],sizeof(self->eobbins)); + InitializeWinZipJPEGContexts(&self->zerobins[0][0][0][0],sizeof(self->zerobins)); + InitializeWinZipJPEGContexts(&self->pivotbins[0][0][0][0],sizeof(self->pivotbins)); + InitializeWinZipJPEGContexts(&self->acmagnitudebins[0][0][0][0][0],sizeof(self->acmagnitudebins)); + InitializeWinZipJPEGContexts(&self->acremainderbins[0][0][0][0],sizeof(self->acremainderbins)); + InitializeWinZipJPEGContexts(&self->acsignbins[0][0][0][0],sizeof(self->acsignbins)); + InitializeWinZipJPEGContexts(&self->dcmagnitudebins[0][0][0],sizeof(self->dcmagnitudebins)); + InitializeWinZipJPEGContexts(&self->dcremainderbins[0][0][0],sizeof(self->dcremainderbins)); + InitializeWinZipJPEGContexts(&self->dcsignbins[0][0][0][0],sizeof(self->dcsignbins)); + + // Calculate slize size, if any. + if(self->slicevalue) + { + int64_t pow2size=1LL<<(self->slicevalue+6); + int64_t div1=pow2size/self->jpeg.horizontalmcus; + if(div1<1) div1=1; + int64_t div2=(self->jpeg.verticalmcus+div1-1)/div1; + self->sliceheight=(unsigned int)((self->jpeg.verticalmcus+div2-1)/div2); + } + else + { + self->sliceheight=self->jpeg.verticalmcus; + } + + // Allocate memory for each component in a slice. + for(int i=0;ijpeg.numscancomponents;i++) + { + self->blocks[i]=malloc(self->jpeg.horizontalmcus*self->sliceheight* + self->jpeg.scancomponents[i].component->horizontalfactor* + self->jpeg.scancomponents[i].component->verticalfactor* + sizeof(WinZipJPEGBlock)); + + if(!self->blocks[i]) return WinZipJPEGOutOfMemoryError; + } + + self->slicesavailable=true; + self->finishedrows=0; + + self->mcucounter=0; + self->restartmarkerindex=0; + self->writerestartmarker=false; + + memset(self->predicted,0,sizeof(self->predicted)); + + self->bitstring=0; + self->bitlength=0; + self->needsstuffing=false; + + return WinZipJPEGNoError; +} + +// Helper function that makes sure to read as much data as requested, even +// if the read function returns short buffers, and reports an error if it +// reaches EOF prematurely. +static int FullRead(WinZipJPEGDecompressor *self,uint8_t *buffer,size_t length) +{ + size_t totalread=0; + while(totalreadreadfunc(self->inputcontext,&buffer[totalread],length-totalread); + if(actual==0) return WinZipJPEGEndOfStreamError; + totalread+=actual; + } + + return WinZipJPEGNoError; +} + +// Helper function to skip data by reading and discarding. +static int SkipBytes(WinZipJPEGDecompressor *self,size_t length) +{ + uint8_t buffer[1024]; + + size_t totalread=0; + while(totalreadsizeof(buffer)) numbytes=sizeof(buffer); + size_t actual=self->readfunc(self->inputcontext,buffer,numbytes); + if(actual==0) return WinZipJPEGEndOfStreamError; + totalread+=actual; + } + + return WinZipJPEGNoError; +} + + + + +// +// Block decoding. +// + +// Decoder functions. +static void DecodeBlock(WinZipJPEGDecompressor *self,int comp, +WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); + +static int DecodeACComponent(WinZipJPEGDecompressor *self,int comp,unsigned int k,bool canbezero, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); + +static int DecodeACSign(WinZipJPEGDecompressor *self,int comp,unsigned int k,int absvalue, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); + +static int DecodeDCComponent(WinZipJPEGDecompressor *self,int comp, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); + +static unsigned int DecodeBinarization(WinZipJPEGArithmeticDecoder *decoder, +WinZipJPEGContext *magnitudebins,WinZipJPEGContext *remainderbins,int maxbits,int cap); + +// Coefficient zig-zag ordering functions. +static bool IsFirstRow(unsigned int k); +static bool IsFirstColumn(unsigned int k); +static bool IsFirstRowOrColumn(unsigned int k); +static bool IsSecondRow(unsigned int k); +static bool IsSecondColumn(unsigned int k); + +static unsigned int Left(unsigned int k); +static unsigned int Up(unsigned int k); +static unsigned int UpAndLeft(unsigned int k); +static unsigned int Right(unsigned int k); +static unsigned int Down(unsigned int k); + +static unsigned int ZigZag(unsigned int row,unsigned int column); +static unsigned int Row(unsigned int k); +static unsigned int Column(unsigned int k); + +// Compression primitive functions. +static int Min(int a,int b); +static int Abs(int x); +static int Sign(int x); +static unsigned int Category(unsigned int val); + +static int Sum(unsigned int k,const WinZipJPEGBlock *block); +static int Average(unsigned int k, +const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); +static int BDR(unsigned int k,const WinZipJPEGBlock *current, +const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization); + +static const WinZipJPEGBlock ZeroBlock={0}; + +int ReadNextWinZipJPEGSlice(WinZipJPEGDecompressor *self) +{ + self->currheight=self->sliceheight; + if(self->finishedrows+self->currheight>=self->jpeg.verticalmcus) + { + self->currheight=self->jpeg.verticalmcus-self->finishedrows; + self->slicesavailable=false; + } + + for(int i=0;ijpeg.numscancomponents;i++) + { + InitializeWinZipJPEGArithmeticDecoder(&self->decoder,self->readfunc,self->inputcontext); + + int hblocks=self->jpeg.scancomponents[i].component->horizontalfactor; + int vblocks=self->jpeg.scancomponents[i].component->verticalfactor; + int blocksperrow=self->jpeg.horizontalmcus*hblocks; + + const WinZipJPEGQuantizationTable *quantization=self->jpeg.scancomponents[i].component->quantizationtable; + + // NOTE: Blocks are processed in cartesian order, not MCU order. + for(int y=0;ycurrheight*vblocks;y++) + for(int x=0;xjpeg.horizontalmcus*hblocks;x++) + { + WinZipJPEGBlock *currblock=&self->blocks[i][x+y*blocksperrow]; + + const WinZipJPEGBlock *northblock; + if(y!=0) northblock=&self->blocks[i][x+(y-1)*blocksperrow]; + else if(self->finishedrows!=0) northblock=&self->blocks[i][x+(self->sliceheight*vblocks-1)*blocksperrow]; + else northblock=NULL; + + const WinZipJPEGBlock *westblock; + if(x!=0) westblock=&self->blocks[i][x-1+y*blocksperrow]; + else westblock=NULL; + + DecodeBlock(self,i,currblock,northblock,westblock,quantization); + + if(WinZipJPEGArithmeticDecoderEncounteredEOF(&self->decoder)) return WinZipJPEGEndOfStreamError; + } + + FlushWinZipJPEGArithmeticDecoder(&self->decoder); + } + + self->finishedrows+=self->currheight; + + // Initialize writer state. + self->mcusavailable=true; + self->currblock=self->blocks[0]; + + self->mcurow=0; + self->mcucol=0; + self->mcucomp=0; + self->mcux=0; + self->mcuy=0; + self->mcucoeff=0; + + // TODO: Error handling. + return WinZipJPEGNoError; +} + +static void DecodeBlock(WinZipJPEGDecompressor *self,int comp, +WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + // Decode End Of Block value to find out how many AC components there are. (5.6.5) + + // Calculate EOB context. (5.6.5.2) + int average; + if(!north&&!west) average=0; + else if(!north) average=Sum(0,west); + else if(!west) average=Sum(0,north); + else average=(Sum(0,north)+Sum(0,west)+1)/2; + + int eobcontext=Min(Category(average),12); + + // Decode EOB bits using binary tree. (5.6.5.1) + unsigned int bitstring=1; + for(int i=0;i<6;i++) + { + bitstring=(bitstring<<1)|NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder, + &self->eobbins[comp][eobcontext][bitstring-1]); + } + unsigned int eob=bitstring&0x3f; + current->eob=eob; + + // Fill out the elided block entries with 0. + for(unsigned int k=eob+1;k<=63;k++) current->c[k]=0; + + // Decode AC components in decreasing order, if any. (5.6.6) + for(unsigned int k=eob;k>=1;k--) + { + current->c[k]=DecodeACComponent(self,comp,k,k!=eob,current,north,west,quantization); + } + + // Decode DC component. + current->c[0]=DecodeDCComponent(self,comp,current,north,west,quantization); +} + +static int DecodeACComponent(WinZipJPEGDecompressor *self,int comp,unsigned int k,bool canbezero, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + if(!north) north=&ZeroBlock; + if(!west) west=&ZeroBlock; + + int val1; + if(IsFirstRowOrColumn(k)) val1=Abs(BDR(k,current,north,west,quantization)); + else val1=Average(k,north,west,quantization); + + int val2=Sum(k,current); + + if(canbezero) + { + // Decode zero/non-zero bit. (5.6.6.1) + int zerocontext1=Min(Category(val1),2); + int zerocontext2=Min(Category(val2),5); + + int nonzero=NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder, + &self->zerobins[comp][k-1][zerocontext1][zerocontext2]); + + // If this component is zero, there is no need to decode further parameters. + if(!nonzero) return 0; + } + + // This component is not zero. Proceed with decoding absolute value. + int absvalue; + + // Decode pivot (abs>=2). (5.6.6.2) + int pivotcontext1=Min(Category(val1),4); + int pivotcontext2=Min(Category(val2),6); + + int pivot=NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder, + &self->pivotbins[comp][k-1][pivotcontext1][pivotcontext2]); + + if(!pivot) + { + // The absolute of this component is not >=2. It must therefore be 1, + // and there is no need to decode the value. + absvalue=1; + } + else + { + // The absolute of this component is >=2. Proceed with decoding + // the absolute value. (5.6.6.3) + int val3,n; + if(IsFirstRow(k)) { val3=Column(k)-1; n=0; } + else if(IsFirstColumn(k)) { val3=Row(k)-1; n=1; } + else { val3=Category(k-4); n=2; } + + int magnitudecontext1=Min(Category(val1),8); + int magnitudecontext2=Min(Category(val2),8); + int remaindercontext=val3; + + // Decode absolute value. + absvalue=DecodeBinarization(&self->decoder, + self->acmagnitudebins[comp][n][magnitudecontext1][magnitudecontext2], + self->acremainderbins[comp][n][remaindercontext], + 14,9)+2; + } + + if(DecodeACSign(self,comp,k,absvalue,current,north,west,quantization)) return -absvalue; + else return absvalue; +} + +static int DecodeACSign(WinZipJPEGDecompressor *self,int comp,unsigned int k,int absvalue, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + // Decode sign. (5.6.6.4) + + // Calculate sign context, or decode with fixed probability. (5.6.6.4.1) + int predictedsign; + if(IsFirstRowOrColumn(k)) + { + int bdr=BDR(k,current,north,west,quantization); + + if(bdr==0) return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder,&self->fixedcontext); + + predictedsign=(bdr<0); + } + else if(k==4) + { + int sign1=Sign(north->c[k]); + int sign2=Sign(west->c[k]); + + if(sign1+sign2==0) return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder,&self->fixedcontext); + + predictedsign=(sign1+sign2<0); + } + else if(IsSecondRow(k)) + { + if(north->c[k]==0) return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder,&self->fixedcontext); + + predictedsign=(north->c[k]<0); + } + else if(IsSecondColumn(k)) + { + if(west->c[k]==0) return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder,&self->fixedcontext); + + predictedsign=(west->c[k]<0); + } + else + { + return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder,&self->fixedcontext); + } + + static const int n_for_k[64]={ + 0, + 0, 1, + 2, 3, 4, + 5, 6, 7, 8, + 9,10, 0,11,12, + 13,14, 0, 0,15,16, + 17,18, 0, 0, 0,19,20, + 21,22, 0, 0, 0, 0,23,24, + 25, 0, 0, 0, 0, 0,26, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, + 0, 0, + 0, + }; + int n=n_for_k[k]; + + int signcontext1=Min(Category(absvalue)/2,2); + + return NextBitFromWinZipJPEGArithmeticDecoder(&self->decoder, + &self->acsignbins[comp][n][signcontext1][predictedsign]); +} + +static int DecodeDCComponent(WinZipJPEGDecompressor *self,int comp, +const WinZipJPEGBlock *current,const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + // Decode DC component. (5.6.7) + + // DC prediction. (5.6.7.1) + int predicted; + if(!north&&!west) + { + predicted=0; + } + else if(!north) + { + // NOTE: Spec says west[1]-current[1]. + int t1=west->c[0]*10000-11038*quantization->c[1]*(west->c[1]+current->c[1])/quantization->c[0]; + int p1=((t1<0)?(t1-5000):(t1+5000))/10000; + predicted=p1; + } + else if(!west) + { + // NOTE: Spec says north->c[2]-current->c[2]. + int t0=north->c[0]*10000-11038*quantization->c[2]*(north->c[2]+current->c[2])/quantization->c[0]; + int p0=((t0<0)?(t0-5000):(t0+5000))/10000; + predicted=p0; + } + else + { + // NOTE: Spec says north[2]-current[2] and west[1]-current[1]. + int t0=north->c[0]*10000-11038*quantization->c[2]*(north->c[2]+current->c[2])/quantization->c[0]; + int p0=((t0<0)?(t0-5000):(t0+5000))/10000; + + int t1=west->c[0]*10000-11038*quantization->c[1]*(west->c[1]+current->c[1])/quantization->c[0]; + int p1=((t1<0)?(t1-5000):(t1+5000))/10000; + + // Prediction refinement. (5.6.7.2) + int d0=0,d1=0; + for(int i=1;i<8;i++) + { + // Note: Spec says Abs(Abs(north->c[ZigZag(i,0)])- + // Abs(current->c[ZigZag(i,0)])) and similarly for west. + d0+=Abs(north->c[ZigZag(i,0)]-current->c[ZigZag(i,0)]); + d1+=Abs(west->c[ZigZag(0,i)]-current->c[ZigZag(0,i)]); + } + + if(d0>d1) + { + int64_t weight=1LL<decoder, + self->dcmagnitudebins[comp][valuecontext], + self->dcremainderbins[comp][valuecontext], + 15,10); + if(absvalue==0) return predicted; + + // Decode sign. (5.6.7.3.2) + // NOTE: Spec says north[0]<0 and west[0]<0. + if(!north) north=&ZeroBlock; + if(!west) west=&ZeroBlock; + int northsign=(north->c[0]c[0]decoder, + &self->dcsignbins[comp][northsign][westsign][predictedsign]); + + if(sign) return predicted-absvalue; + else return predicted+absvalue; +} + +static unsigned int DecodeBinarization(WinZipJPEGArithmeticDecoder *decoder, +WinZipJPEGContext *magnitudebins,WinZipJPEGContext *remainderbins,int maxbits,int cap) +{ + // Decode binarization. (5.6.4, and additional reverse engineering + // as the spec does not describe the process in sufficient detail.) + + // Decode unary magnitude. + int ones=0; + while(ones=cap) context=cap-1; + + int unary=NextBitFromWinZipJPEGArithmeticDecoder(decoder,&magnitudebins[context]); + if(unary==1) ones++; + else break; + } + + // Decode remainder bits, if any. + if(ones==0) return 0; + else if(ones==1) return 1; + else + { + int numbits=ones-1; + int val=1<=0;i--) + { + int bit=NextBitFromWinZipJPEGArithmeticDecoder(decoder,&remainderbins[i]); + val|=bit<=8||column>=8) return 0; // Can't happen. + static const int table[8][8]= + { + { 0, 1, 5, 6,14,15,27,28, }, + { 2, 4, 7,13,16,26,29,42, }, + { 3, 8,12,17,25,30,41,43, }, + { 9,11,18,24,31,40,44,53, }, + { 10,19,23,32,39,45,52,54, }, + { 20,22,33,38,46,51,55,60, }, + { 21,34,37,47,50,56,59,61, }, + { 35,36,48,49,57,58,62,63, }, + }; + return table[row][column]; +} + +static unsigned int Row(unsigned int k) +{ + if(k>=64) return 0; // Can't happen. + static const int table[64]= + { + 0,0,1,2,1,0,0,1,2,3,4,3,2,1,0,0, + 1,2,3,4,5,6,5,4,3,2,1,0,0,1,2,3, + 4,5,6,7,7,6,5,4,3,2,1,2,3,4,5,6, + 7,7,6,5,4,3,4,5,6,7,7,6,5,6,7,7, + }; + return table[k]; +} + +static unsigned int Column(unsigned int k) +{ + if(k>=64) return 0; // Can't happen. + static const int table[64]= + { + 0,1,0,0,1,2,3,2,1,0,0,1,2,3,4,5, + 4,3,2,1,0,0,1,2,3,4,5,6,7,6,5,4, + 3,2,1,0,1,2,3,4,5,6,7,7,6,5,4,3, + 2,3,4,5,6,7,7,6,5,4,5,6,7,7,6,7, + }; + return table[k]; +} + +static int Min(int a,int b) +{ + if(a=0) return x; + else return -x; +} + +static int Sign(int x) +{ + if(x>0) return 1; + else if(x<0) return -1; + else return 0; +} + +// CAT (5.6.3) +static unsigned int Category(unsigned int val) +{ + if(val==0) return 0; + + unsigned int cat=0; + if(val&0xffff0000) { val>>=16; cat|=16; } + if(val&0xff00) { val>>=8; cat|=8; } + if(val&0xf0) { val>>=4; cat|=4; } + if(val&0xc) { val>>=2; cat|=2; } + if(val&0x2) { /*val>>=1;*/ cat|=1; } + return cat+1; +} + +// SUM (5.6.2.1) +static int Sum(unsigned int k,const WinZipJPEGBlock *block) +{ + int sum=0; + for(unsigned int i=0;i<64;i++) + { + if(i!=k && Row(i)>=Row(k) && Column(i)>=Column(k)) + sum+=Abs(block->c[i]); + } + return sum; +} + +// AVG (5.6.2.2) +// NOTE: This assumes that the expression given for 'sum' is incorrect, and that +// Bw[k] should actually be Bw[x]. Also, the spec does not explicitly mention +// that the DC component never contributes. +static int Average(unsigned int k, +const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + if(k==0||k==1||k==2) + { + return (Abs(north->c[k])+Abs(west->c[k])+1)/2; + } + else if(IsFirstRow(k)) + { + return ( + (Abs(north->c[Left(k)])+Abs(west->c[Left(k)]))*quantization->c[Left(k)]/quantization->c[k]+ + Abs(north->c[k])+Abs(west->c[k])+ + 2 + )/(2*2); + } + else if(IsFirstColumn(k)) + { + return ( + (Abs(north->c[Up(k)])+Abs(west->c[Up(k)]))*quantization->c[Up(k)]/quantization->c[k]+ + Abs(north->c[k])+Abs(west->c[k])+ + 2 + )/(2*2); + } + else if(k==4) + { + return ( + (Abs(north->c[Up(k)])+Abs(west->c[Up(k)]))*quantization->c[Up(k)]/quantization->c[k]+ + (Abs(north->c[Left(k)])+Abs(west->c[Left(k)]))*quantization->c[Left(k)]/quantization->c[k]+ + Abs(north->c[k])+Abs(west->c[k])+ + 3 + )/(2*3); + } + else + { + return ( + (Abs(north->c[Up(k)])+Abs(west->c[Up(k)]))*quantization->c[Up(k)]/quantization->c[k]+ + (Abs(north->c[Left(k)])+Abs(west->c[Left(k)]))*quantization->c[Left(k)]/quantization->c[k]+ + (Abs(north->c[UpAndLeft(k)])+Abs(west->c[UpAndLeft(k)]))*quantization->c[UpAndLeft(k)]/quantization->c[k]+ + Abs(north->c[k])+Abs(west->c[k])+ + 4 + )/(2*4); + } +} + +// BDR (5.6.2.3) +static int BDR(unsigned int k,const WinZipJPEGBlock *current, +const WinZipJPEGBlock *north,const WinZipJPEGBlock *west, +const WinZipJPEGQuantizationTable *quantization) +{ + if(IsFirstRow(k)) + { + return north->c[k]-(north->c[Down(k)]+current->c[Down(k)])*quantization->c[Down(k)]/quantization->c[k]; + } + else if(IsFirstColumn(k)) + { + return west->c[k]-(west->c[Right(k)]+current->c[Right(k)])*quantization->c[Right(k)]/quantization->c[k]; + } + else return 0; // Can't happen. +} + + + +// +// Block encoding. +// + +static void PushEncodedValue(WinZipJPEGDecompressor *self,WinZipJPEGHuffmanTable *table, +int value,unsigned int highbits); +static void PushHuffmanCode(WinZipJPEGDecompressor *self,WinZipJPEGHuffmanTable *table,unsigned int code); +static void PushBitString(WinZipJPEGDecompressor *self,uint32_t bitstring,unsigned int length); + +size_t EncodeWinZipJPEGBlocksToBuffer(WinZipJPEGDecompressor *self,void *bytes,size_t length) +{ + uint8_t *start=bytes; + uint8_t *ptr=bytes; + uint8_t *end=ptr+length; + + while(ptrneedsstuffing) + { + // If we need to add a byte of stuffing, do so. + *ptr++=0x00; + self->needsstuffing=false; + } + else if(self->bitlength>=8) + { + // If there are enough buffered bits, output one byte. + uint8_t byte=self->bitstring>>56LL; + *ptr++=byte; + self->bitstring<<=8; + self->bitlength-=8; + + if(byte==0xff) self->needsstuffing=true; + } + else if(self->writerestartmarker) + { + // Output the first half of a restart marker. This has to be done + // separately, to avoid stuffing. + *ptr++=0xff; + + // Next, push the rest of the marker into the bitstream as usual, as it + // will not trigger bit stuffing. + PushBitString(self,0xd0+self->restartmarkerindex,8); + + // Cycle the restart marker indexes, reset the MCU counter, and clear predictors. + self->restartmarkerindex=(self->restartmarkerindex+1)&7; + self->mcucounter=0; + memset(self->predicted,0,sizeof(self->predicted)); + + self->writerestartmarker=false; + } + else if(self->jpeg.restartinterval && self->mcucounter==self->jpeg.restartinterval && + (self->mcusavailable || self->slicesavailable)) + { + // If it is time for a restart marker, and if we have not reached the very + // end of the scan, start outputting one. + + // First, pad with ones to a byte border if needed. + if(self->bitlength) + { + int n=8-self->bitlength; + PushBitString(self,(1<writerestartmarker=true; + } + else if(self->mcusavailable) + { + // If there are still MCUs left to process output either a DC or AC coefficient as appropriate. + if(self->mcucoeff==0) + { + // Output DC coefficient. + int diff=self->currblock->c[0]-self->predicted[self->mcucomp]; + + PushEncodedValue(self,self->jpeg.scancomponents[self->mcucomp].dctable,diff,0); + + self->predicted[self->mcucomp]=self->currblock->c[0]; + self->mcucoeff=1; + } + else if(self->mcucoeff>self->currblock->eob && self->currblock->eob!=63) + { + // Output EOB marker. + PushHuffmanCode(self,self->jpeg.scancomponents[self->mcucomp].actable,0x00); + + self->mcucoeff=64; + } + else + { + // Output AC coefficient. + + // Find the next non-zero coefficient. + int firstcoeff=self->mcucoeff; + int endrun=self->mcucoeff+15; + while(self->mcucoeff<63 && self->mcucoeffcurrblock->c[self->mcucoeff]==0) self->mcucoeff++; + + int zeroes=self->mcucoeff-firstcoeff; + int val=self->currblock->c[self->mcucoeff]; + + PushEncodedValue(self,self->jpeg.scancomponents[self->mcucomp].actable,val,zeroes); + + self->mcucoeff++; + } + + // If we have output all coefficients, update position. + if(self->mcucoeff>=64) + { + int hblocks=self->jpeg.scancomponents[self->mcucomp].component->horizontalfactor; + int vblocks=self->jpeg.scancomponents[self->mcucomp].component->verticalfactor; + + self->mcucoeff=0; self->mcux++; + if(self->mcux>=hblocks) + { + self->mcux=0; self->mcuy++; + if(self->mcuy>=vblocks) + { + self->mcuy=0; self->mcucomp++; + if(self->mcucomp>=self->jpeg.numscancomponents) + { + self->mcucomp=0; self->mcucol++; + if(self->mcucol>=self->jpeg.horizontalmcus) + { + self->mcucol=0; self->mcurow++; + if(self->mcurow>=self->currheight) + { + self->mcusavailable=false; + + if(!self->slicesavailable) + { + // If we reached the very end, pad with ones + // to a byte boundary to finish the stream. + int n=(-self->bitlength)&7; + PushBitString(self,(1<mcucounter++; + } + } + } + + // Find the new block. + hblocks=self->jpeg.scancomponents[self->mcucomp].component->horizontalfactor; + vblocks=self->jpeg.scancomponents[self->mcucomp].component->verticalfactor; + int blocksperrow=self->jpeg.horizontalmcus*hblocks; + + int blockx=self->mcucol*hblocks+self->mcux; + int blocky=self->mcurow*vblocks+self->mcuy; + + self->currblock=&self->blocks[self->mcucomp][blockx+blocky*blocksperrow]; + } + } + else + { + // Nothing left to do. Return the partial length of output data. + return ptr-start; + } + } + return length; +} + +static void PushEncodedValue(WinZipJPEGDecompressor *self,WinZipJPEGHuffmanTable *table, +int value,unsigned int highbits) +{ + unsigned int category,bitstring; + if(value>=0) + { + category=Category(value); + analyser_assert(category>=1 && category<=32); + unsigned int mask=(1ull<=1 && category<=32); + unsigned int mask=(1ull<codes[code].code,table->codes[code].length); +} + +static void PushBitString(WinZipJPEGDecompressor *self,uint32_t bitstring,unsigned int length) +{ + self->bitstring|=(uint64_t)bitstring<<(64-self->bitlength-length); + self->bitlength+=length; +} diff --git a/winzipjpeg/Decompressor.h b/winzipjpeg/Decompressor.h new file mode 100644 index 0000000..558d113 --- /dev/null +++ b/winzipjpeg/Decompressor.h @@ -0,0 +1,102 @@ +/* + * Decompressor.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __WINZIP_JPEG_DECOMPRESSOR_H__ +#define __WINZIP_JPEG_DECOMPRESSOR_H__ + +#include "ArithmeticDecoder.h" +#include "InputStream.h" +#include "JPEG.h" + +#include +#include + +#define WinZipJPEGNoError 0 +#define WinZipJPEGEndOfStreamError 1 +#define WinZipJPEGOutOfMemoryError 2 +#define WinZipJPEGInvalidHeaderError 3 +#define WinZipJPEGLZMAError 4 +#define WinZipJPEGParseError 5 + +typedef struct WinZipJPEGDecompressor +{ + WinZipJPEGReadFunction *readfunc; + void *inputcontext; + + uint32_t metadatalength; + uint8_t *metadatabytes; + + bool isfirstbundle, reachedend; + WinZipJPEGMetadata jpeg; + + bool slicesavailable; + unsigned int slicevalue, sliceheight; + unsigned int currheight, finishedrows; + + int predicted[4]; + + uint64_t bitstring; + unsigned int bitlength; + bool needsstuffing; + + WinZipJPEGArithmeticDecoder decoder; + + WinZipJPEGContext eobbins[4][13][63]; // 321 in WinZip. + WinZipJPEGContext zerobins[4][62][3][6]; // 1140 in WinZip. + WinZipJPEGContext pivotbins[4][63][5][7]; // 2256 in WinZip. + WinZipJPEGContext acmagnitudebins[4][3][9][9][9]; + WinZipJPEGContext acremainderbins[4][3][7][13]; + WinZipJPEGContext acsignbins[4][27][3][2]; + WinZipJPEGContext dcmagnitudebins[4][13][10]; // 1 in WinZip. + WinZipJPEGContext dcremainderbins[4][13][14]; // 131 in WinZip. + WinZipJPEGContext dcsignbins[4][2][2][2]; // 313 in WinZip. + WinZipJPEGContext fixedcontext; // 0 in WinZip. + + WinZipJPEGBlock *blocks[4]; + + WinZipJPEGBlock *currblock; + bool mcusavailable; + unsigned int mcurow, mcucol, mcucomp, mcux, mcuy, mcucoeff; + unsigned int mcucounter, restartmarkerindex; + bool writerestartmarker; +} WinZipJPEGDecompressor; + +WinZipJPEGDecompressor *AllocWinZipJPEGDecompressor(WinZipJPEGReadFunction *readfunc, void *inputcontext); +void FreeWinZipJPEGDecompressor(WinZipJPEGDecompressor *self); + +int ReadWinZipJPEGHeader(WinZipJPEGDecompressor *self); +int ReadNextWinZipJPEGBundle(WinZipJPEGDecompressor *self); +int ReadNextWinZipJPEGSlice(WinZipJPEGDecompressor *self); + +size_t EncodeWinZipJPEGBlocksToBuffer(WinZipJPEGDecompressor *self, void *bytes, size_t length); + +static inline bool IsFinalWinZipJPEGBundle(WinZipJPEGDecompressor *self) { return self->reachedend; } + +static inline bool AreMoreWinZipJPEGSlicesAvailable(WinZipJPEGDecompressor *self) +{ return !self->reachedend && self->slicesavailable; } + +static inline bool AreMoreWinZipJPEGBytesAvailable(WinZipJPEGDecompressor *self) +{ return self->mcusavailable || self->bitlength >= 8 || self->needsstuffing || self->writerestartmarker; } + +static inline uint32_t WinZipJPEGBundleMetadataLength(WinZipJPEGDecompressor *self) { return self->metadatalength; } + +static inline uint8_t *WinZipJPEGBundleMetadataBytes(WinZipJPEGDecompressor *self) { return self->metadatabytes; } + +#endif diff --git a/winzipjpeg/InputStream.h b/winzipjpeg/InputStream.h new file mode 100644 index 0000000..0e487e6 --- /dev/null +++ b/winzipjpeg/InputStream.h @@ -0,0 +1,29 @@ +/* + * InputStream.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __WINZIP_JPEG_INPUT_STREAM_H__ +#define __WINZIP_JPEG_INPUT_STREAM_H__ + +#include +#include + +typedef size_t WinZipJPEGReadFunction(void *context, uint8_t *buffer, size_t length); + +#endif diff --git a/winzipjpeg/JPEG.c b/winzipjpeg/JPEG.c new file mode 100644 index 0000000..e0be058 --- /dev/null +++ b/winzipjpeg/JPEG.c @@ -0,0 +1,329 @@ +/* + * JPEG.c + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#include "JPEG.h" + +#include + +// #include +// #define DebugPrint(...) fprintf(stderr,__VA_ARGS__) +#define DebugPrint(...) + +static const uint8_t *FindNextMarker(const uint8_t *ptr, const uint8_t *end); +static int ParseSize(const uint8_t *ptr, const uint8_t *end); + +static inline uint16_t ParseUInt16(const uint8_t *ptr) { return (ptr[0] << 8) | ptr[1]; } + +const void *FindStartOfWinZipJPEGImage(const void *bytes, size_t length) +{ + const uint8_t *ptr = bytes; + const uint8_t *end = ptr + length; + + while(ptr + 2 <= end) + { + if(ptr[0] == 0xff && ptr[1] == 0xd8) return ptr; + ptr++; + } + + return NULL; +} + +void InitializeWinZipJPEGMetadata(WinZipJPEGMetadata *self) { memset(self, 0, sizeof(*self)); } + +int ParseWinZipJPEGMetadata(WinZipJPEGMetadata *self, const void *bytes, size_t length) +{ + const uint8_t *ptr = bytes; + const uint8_t *end = ptr + length; + + for(;;) + { + ptr = FindNextMarker(ptr, end); + if(!ptr) return WinZipJPEGMetadataParsingFailed; + + switch(*ptr++) + { + case 0xd8: // Start of image + // Empty marker, do nothing. + DebugPrint("Start of image\n"); + break; + + case 0xc4: // Define huffman table + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + const uint8_t *next = ptr + size; + + ptr += 2; + + DebugPrint("Define huffman table(s)\n"); + + while(ptr + 17 <= next) + { + int class = *ptr >> 4; + int index = *ptr & 0x0f; + ptr++; + + if(class != 0 && class != 1) return WinZipJPEGMetadataParsingFailed; + if(index >= 4) return WinZipJPEGMetadataParsingFailed; + + int numcodes[16]; + int totalcodes = 0; + for(int i = 0; i < 16; i++) + { + numcodes[i] = ptr[i]; + totalcodes += numcodes[i]; + } + ptr += 16; + + if(ptr + totalcodes > next) return WinZipJPEGMetadataParsingFailed; + + DebugPrint(" > %s table at %d with %d codes\n", class == 0 ? "DC" : "AC", index, totalcodes); + + unsigned int code = 0; + for(int i = 0; i < 16; i++) + { + for(int j = 0; j < numcodes[i]; j++) + { + int value = *ptr++; + + self->huffmantables[class][index].codes[value].code = code; + self->huffmantables[class][index].codes[value].length = i + 1; + // DebugPrint(" >> Code %x length %d for %d\n",code,i+1,value); + + code++; + } + + code <<= 1; + } + } + + ptr = next; + } + break; + + case 0xdb: // Define quantization table(s) + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + const uint8_t *next = ptr + size; + + ptr += 2; + + DebugPrint("Define quantization table(s)\n"); + + while(ptr + 1 <= next) + { + int precision = *ptr >> 4; + int index = *ptr & 0x0f; + ptr++; + + if(index >= 4) return WinZipJPEGMetadataParsingFailed; + + if(precision == 0) + { + DebugPrint(" > 8 bit table at %d\n", index); + + if(ptr + 64 > next) return WinZipJPEGMetadataParsingFailed; + for(int i = 0; i < 64; i++) self->quantizationtables[index].c[i] = ptr[i]; + ptr += 64; + } + else if(precision == 1) + { + DebugPrint(" > 16 bit table at %d\n", index); + + if(ptr + 128 > next) return WinZipJPEGMetadataParsingFailed; + for(int i = 0; i < 64; i++) self->quantizationtables[index].c[i] = ParseUInt16(&ptr[2 * i]); + ptr += 128; + } + else + return WinZipJPEGMetadataParsingFailed; + } + + ptr = next; + } + break; + + case 0xdd: // Define restart interval + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + const uint8_t *next = ptr + size; + + self->restartinterval = ParseUInt16(&ptr[2]); + + ptr = next; + + DebugPrint("Define restart interval: %d\n", self->restartinterval); + } + break; + + case 0xc0: // Start of frame 0 + case 0xc1: // Start of frame 1 + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + const uint8_t *next = ptr + size; + + if(size < 8) return WinZipJPEGMetadataParsingFailed; + self->bits = ptr[2]; + self->height = ParseUInt16(&ptr[3]); + self->width = ParseUInt16(&ptr[5]); + self->numcomponents = ptr[7]; + + if(self->numcomponents < 1 || self->numcomponents > 4) return WinZipJPEGMetadataParsingFailed; + if(size < 8 + self->numcomponents * 3) return WinZipJPEGMetadataParsingFailed; + + self->maxhorizontalfactor = 1; + self->maxverticalfactor = 1; + + DebugPrint("Start of frame: %dx%d %d bits %d comps\n", self->width, self->height, self->bits, + self->numcomponents); + + for(int i = 0; i < self->numcomponents; i++) + { + self->components[i].identifier = ptr[8 + i * 3]; + self->components[i].horizontalfactor = ptr[9 + i * 3] >> 4; + self->components[i].verticalfactor = ptr[9 + i * 3] & 0x0f; + int quantizationindex = ptr[10 + i * 3]; + self->components[i].quantizationtable = &self->quantizationtables[quantizationindex]; + + if(self->components[i].horizontalfactor > self->maxhorizontalfactor) + self->maxhorizontalfactor = self->components[i].horizontalfactor; + + if(self->components[i].verticalfactor > self->maxverticalfactor) + self->maxverticalfactor = self->components[i].verticalfactor; + + DebugPrint(" > Component id %d, %dx%d, quant %d\n", self->components[i].identifier, + self->components[i].horizontalfactor, self->components[i].verticalfactor, + quantizationindex); + } + + // TODO: This is a kludge for strange one-component files with + // 2x2 sampling factor, that are still stored in exactly the same + // way as 1x1. Figure out how to actually handle this properly. + if(self->numcomponents == 1) + { + self->components[0].horizontalfactor /= self->maxhorizontalfactor; + self->components[0].verticalfactor /= self->maxverticalfactor; + self->maxhorizontalfactor = 1; + self->maxverticalfactor = 1; + } + + int mcuwidth = self->maxhorizontalfactor * 8; + int mcuheight = self->maxverticalfactor * 8; + self->horizontalmcus = (self->width + mcuwidth - 1) / mcuwidth; + self->verticalmcus = (self->height + mcuheight - 1) / mcuheight; + + DebugPrint(" > MCU size %dx%d, %d horizontal MCUs, %d vertical MCUs.\n", mcuwidth, mcuheight, + self->horizontalmcus, self->verticalmcus); + + ptr = next; + } + break; + + case 0xda: // Start of scan + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + + if(size < 6) return WinZipJPEGMetadataParsingFailed; + + self->numscancomponents = ptr[2]; + if(self->numscancomponents < 1 || self->numscancomponents > 4) return WinZipJPEGMetadataParsingFailed; + if(size < 6 + self->numscancomponents * 2) return WinZipJPEGMetadataParsingFailed; + + DebugPrint("Start of scan: %d comps\n", self->numscancomponents); + + for(int i = 0; i < self->numscancomponents; i++) + { + int identifier = ptr[3 + i * 2]; + WinZipJPEGComponent *component = NULL; + for(int j = 0; j < self->numcomponents; j++) + { + if(self->components[j].identifier == identifier) + { + component = &self->components[j]; + break; + } + } + if(!component) return WinZipJPEGMetadataParsingFailed; + + self->scancomponents[i].component = component; + + int dcindex = ptr[4 + i * 2] >> 4; + int acindex = ptr[4 + i * 2] & 0x0f; + self->scancomponents[i].dctable = &self->huffmantables[0][dcindex]; + self->scancomponents[i].actable = &self->huffmantables[1][acindex]; + + DebugPrint(" > Component id %d, %dx%d, DC %d, AC %d\n", identifier, + self->scancomponents[i].component->horizontalfactor, + self->scancomponents[i].component->verticalfactor, dcindex, acindex); + } + + if(ptr[3 + self->numscancomponents * 2] != 0) return WinZipJPEGMetadataParsingFailed; + if(ptr[4 + self->numscancomponents * 2] != 63) return WinZipJPEGMetadataParsingFailed; + if(ptr[5 + self->numscancomponents * 2] != 0) return WinZipJPEGMetadataParsingFailed; + + return WinZipJPEGMetadataFoundStartOfScan; + } + break; + + case 0xd9: // End of image + return WinZipJPEGMetadataFoundEndOfImage; + + default: + { + int size = ParseSize(ptr, end); + if(!size) return WinZipJPEGMetadataParsingFailed; + ptr += size; + + DebugPrint("Unknown marker %02x\n", ptr[-1]); + } + break; + } + } +} + +// Find next marker, skipping pad bytes. +static const uint8_t *FindNextMarker(const uint8_t *ptr, const uint8_t *end) +{ + if(ptr >= end) return NULL; + if(*ptr != 0xff) return NULL; + + while(*ptr == 0xff) + { + ptr++; + if(ptr >= end) return NULL; + } + + return ptr; +} + +// Parse and sanity check the size of a marker. +static int ParseSize(const uint8_t *ptr, const uint8_t *end) +{ + if(ptr + 2 > end) return 0; + + int size = ParseUInt16(ptr); + if(size < 2) return 0; + if(ptr + size > end) return 0; + + return size; +} diff --git a/winzipjpeg/JPEG.h b/winzipjpeg/JPEG.h new file mode 100644 index 0000000..fa1c561 --- /dev/null +++ b/winzipjpeg/JPEG.h @@ -0,0 +1,89 @@ +/* + * JPEG.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __WINZIP_JPEG_JPEG_H__ +#define __WINZIP_JPEG_JPEG_H__ + +#include +#include +#include + +#define WinZipJPEGMetadataFoundStartOfScan 1 +#define WinZipJPEGMetadataFoundEndOfImage 2 +#define WinZipJPEGMetadataParsingFailed 3 + +typedef struct WinZipJPEGBlock +{ + int16_t c[64]; + uint8_t eob; +} WinZipJPEGBlock; + +typedef struct WinZipJPEGQuantizationTable +{ + int16_t c[64]; +} WinZipJPEGQuantizationTable; + +typedef struct WinZipJPEGHuffmanCode +{ + unsigned int code, length; +} WinZipJPEGHuffmanCode; + +typedef struct WinZipJPEGHuffmanTable +{ + WinZipJPEGHuffmanCode codes[256]; +} WinZipJPEGHuffmanTable; + +typedef struct WinZipJPEGComponent +{ + unsigned int identifier; + unsigned int horizontalfactor, verticalfactor; + WinZipJPEGQuantizationTable *quantizationtable; +} WinZipJPEGComponent; + +typedef struct WinZipJPEGScanComponent +{ + WinZipJPEGComponent *component; + WinZipJPEGHuffmanTable *dctable, *actable; +} WinZipJPEGScanComponent; + +typedef struct WinZipJPEGMetadata +{ + unsigned int width, height, bits; + unsigned int restartinterval; + + unsigned int maxhorizontalfactor, maxverticalfactor; + unsigned int horizontalmcus, verticalmcus; + + unsigned int numcomponents; + WinZipJPEGComponent components[4]; + + unsigned int numscancomponents; + WinZipJPEGScanComponent scancomponents[4]; + + WinZipJPEGQuantizationTable quantizationtables[4]; + WinZipJPEGHuffmanTable huffmantables[2][4]; +} WinZipJPEGMetadata; + +const void *FindStartOfWinZipJPEGImage(const void *bytes, size_t length); + +void InitializeWinZipJPEGMetadata(WinZipJPEGMetadata *self); +int ParseWinZipJPEGMetadata(WinZipJPEGMetadata *self, const void *bytes, size_t length); + +#endif diff --git a/winzipjpeg/LZMA.h b/winzipjpeg/LZMA.h new file mode 100644 index 0000000..9977044 --- /dev/null +++ b/winzipjpeg/LZMA.h @@ -0,0 +1,32 @@ +/* + * LZMA.h + * + * Copyright (c) 2017-present, MacPaw Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ +#ifndef __WINZIP_JPEG_LZMA_H__ +#define __WINZIP_JPEG_LZMA_H__ + +#if !__LP64__ +#define _LZMA_UINT32_IS_ULONG +#endif + +#define Byte LzmaByte +#include "../3rdparty/lzma/C/LzmaDec.h" +#undef Byte + +#endif diff --git a/zip/deflate64.c b/zip/deflate64.c new file mode 100644 index 0000000..620c68c --- /dev/null +++ b/zip/deflate64.c @@ -0,0 +1,350 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "deflate64.h" + +#include +#include + +#include "../pak/bitstream.h" +#include "../pak/prefixcode.h" + +/* Deflate64 window size: 64KB */ +#define DEFLATE64_WINDOW_SIZE 65536 +#define DEFLATE64_WINDOW_MASK (DEFLATE64_WINDOW_SIZE - 1) + +/* Block types */ +#define BLOCK_STORED 0 +#define BLOCK_FIXED 1 +#define BLOCK_DYNAMIC 2 + +/* Number of symbols */ +#define NUM_LITLEN_SYMBOLS 288 +#define NUM_DIST_SYMBOLS 32 +#define MAX_LITLEN_SYMBOLS 286 /* Only 0-285 are defined for Deflate64 */ + +/* Length base values for codes 257-285 */ +static const int length_base[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, + 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 3}; + +/* Length extra bits for codes 257-285 (Deflate64: code 285 has 16 extra bits) */ +static const int length_extra[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, + 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 16}; + +/* Distance base values for codes 0-31 */ +static const int dist_base[32] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, + 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, + 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 32769, 49153}; + +/* Distance extra bits for codes 0-31 */ +static const int dist_extra[32] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14}; + +/* Order of code length codes for dynamic Huffman tables */ +static const int codelen_order[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + +/* Build static fixed Huffman tables for Deflate */ +static PrefixCode *build_fixed_litlen_code(void) +{ + int lengths[NUM_LITLEN_SYMBOLS]; + int i; + + for(i = 0; i <= 143; i++) lengths[i] = 8; + for(i = 144; i <= 255; i++) lengths[i] = 9; + for(i = 256; i <= 279; i++) lengths[i] = 7; + for(i = 280; i <= 287; i++) lengths[i] = 8; + + return prefix_code_alloc_with_lengths(lengths, NUM_LITLEN_SYMBOLS, 9, true); +} + +static PrefixCode *build_fixed_dist_code(void) +{ + int lengths[NUM_DIST_SYMBOLS]; + + for(int i = 0; i < NUM_DIST_SYMBOLS; i++) lengths[i] = 5; + + return prefix_code_alloc_with_lengths(lengths, NUM_DIST_SYMBOLS, 5, true); +} + +/* Build dynamic Huffman tables from the stream */ +static int build_dynamic_tables(BitStream *bs, PrefixCode **litlen_code_out, PrefixCode **dist_code_out) +{ + int hlit = (int)bitstream_read_bits_le(bs, 5) + 257; + int hdist = (int)bitstream_read_bits_le(bs, 5) + 1; + int hclen = (int)bitstream_read_bits_le(bs, 4) + 4; + + /* Read code length code lengths */ + int codelen_lengths[19]; + memset(codelen_lengths, 0, sizeof(codelen_lengths)); + + for(int i = 0; i < hclen; i++) codelen_lengths[codelen_order[i]] = (int)bitstream_read_bits_le(bs, 3); + + /* Build code length code */ + PrefixCode *codelen_code = prefix_code_alloc_with_lengths(codelen_lengths, 19, 7, true); + + if(!codelen_code) return -1; + + /* Read literal/length and distance code lengths */ + int total_codes = hlit + hdist; + int *all_lengths = (int *)calloc(total_codes, sizeof(int)); + + if(!all_lengths) + { + prefix_code_free(codelen_code); + return -1; + } + + int idx = 0; + + while(idx < total_codes) + { + int sym = prefix_code_read_symbol_le(bs, codelen_code); + + if(sym < 0) + { + free(all_lengths); + prefix_code_free(codelen_code); + return -1; + } + + if(sym < 16) + { + /* Literal length value */ + all_lengths[idx++] = sym; + } + else if(sym == 16) + { + /* Repeat previous length 3-6 times */ + int repeat = (int)bitstream_read_bits_le(bs, 2) + 3; + int prev = idx > 0 ? all_lengths[idx - 1] : 0; + + for(int i = 0; i < repeat && idx < total_codes; i++) all_lengths[idx++] = prev; + } + else if(sym == 17) + { + /* Repeat zero 3-10 times */ + int repeat = (int)bitstream_read_bits_le(bs, 3) + 3; + + for(int i = 0; i < repeat && idx < total_codes; i++) all_lengths[idx++] = 0; + } + else if(sym == 18) + { + /* Repeat zero 11-138 times */ + int repeat = (int)bitstream_read_bits_le(bs, 7) + 11; + + for(int i = 0; i < repeat && idx < total_codes; i++) all_lengths[idx++] = 0; + } + } + + prefix_code_free(codelen_code); + + /* Find max code lengths */ + int max_litlen = 0; + + for(int i = 0; i < hlit; i++) + { + if(all_lengths[i] > max_litlen) max_litlen = all_lengths[i]; + } + + if(max_litlen == 0) max_litlen = 1; + + int max_dist = 0; + + for(int i = hlit; i < total_codes; i++) + { + if(all_lengths[i] > max_dist) max_dist = all_lengths[i]; + } + + if(max_dist == 0) max_dist = 1; + + /* Build literal/length code */ + *litlen_code_out = prefix_code_alloc_with_lengths(all_lengths, hlit, max_litlen, true); + + /* Build distance code */ + *dist_code_out = prefix_code_alloc_with_lengths(all_lengths + hlit, hdist, max_dist, true); + + free(all_lengths); + + if(!*litlen_code_out || !*dist_code_out) + { + if(*litlen_code_out) + { + prefix_code_free(*litlen_code_out); + *litlen_code_out = NULL; + } + + if(*dist_code_out) + { + prefix_code_free(*dist_code_out); + *dist_code_out = NULL; + } + + return -1; + } + + return 0; +} + +int zip_deflate64_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len) +{ + BitStream bs; + size_t out_pos = 0; + size_t out_size = *out_len; + int result = 0; + int is_final = 0; + + if(!in_buf || !out_buf || !out_len) return -1; + + bitstream_init(&bs, in_buf, in_len); + + while(!is_final && out_pos < out_size) + { + PrefixCode *litlen_code = NULL; + PrefixCode *dist_code = NULL; + int free_codes = 0; + + is_final = (int)bitstream_read_bits_le(&bs, 1); + int block_type = (int)bitstream_read_bits_le(&bs, 2); + + if(block_type == BLOCK_STORED) + { + /* Skip to byte boundary */ + if(bs.bitcount > 0) + { + bs.bitbuffer = 0; + bs.bitcount = 0; + } + + uint16_t len = bitstream_read_uint16_le(&bs); + uint16_t nlen = bitstream_read_uint16_le(&bs); + + (void)nlen; /* Complement check can be skipped; we trust the data */ + + for(uint16_t i = 0; i < len && out_pos < out_size; i++) + { + if(bs.pos >= bs.length) break; + + out_buf[out_pos++] = bs.data[bs.pos++]; + } + + continue; + } + else if(block_type == BLOCK_FIXED) + { + litlen_code = build_fixed_litlen_code(); + dist_code = build_fixed_dist_code(); + free_codes = 1; + + if(!litlen_code || !dist_code) + { + result = -1; + goto block_cleanup; + } + } + else if(block_type == BLOCK_DYNAMIC) + { + if(build_dynamic_tables(&bs, &litlen_code, &dist_code) != 0) + { + result = -1; + goto block_cleanup; + } + + free_codes = 1; + } + else + { + result = -1; + break; + } + + /* Decode symbols */ + while(out_pos < out_size) + { + int sym = prefix_code_read_symbol_le(&bs, litlen_code); + + if(sym < 0) + { + result = -1; + goto block_cleanup; + } + + if(sym < 256) + { + /* Literal byte */ + out_buf[out_pos++] = (uint8_t)sym; + } + else if(sym == 256) + { + /* End of block */ + break; + } + else + { + /* Length/distance pair */ + int length_code_idx = sym - 257; + + if(length_code_idx < 0 || length_code_idx >= 29) + { + result = -1; + goto block_cleanup; + } + + int length = length_base[length_code_idx]; + + if(length_extra[length_code_idx] > 0) + length += (int)bitstream_read_bits_le(&bs, length_extra[length_code_idx]); + + /* Read distance */ + int dist_sym = prefix_code_read_symbol_le(&bs, dist_code); + + if(dist_sym < 0 || dist_sym >= 32) + { + result = -1; + goto block_cleanup; + } + + int distance = dist_base[dist_sym]; + + if(dist_extra[dist_sym] > 0) distance += (int)bitstream_read_bits_le(&bs, dist_extra[dist_sym]); + + /* Copy from history */ + for(int i = 0; i < length && out_pos < out_size; i++) + { + if((size_t)distance > out_pos) + out_buf[out_pos] = 0; + else + out_buf[out_pos] = out_buf[out_pos - distance]; + + out_pos++; + } + } + } + + block_cleanup: + if(free_codes) + { + if(litlen_code) prefix_code_free(litlen_code); + if(dist_code) prefix_code_free(dist_code); + } + + if(result != 0) break; + } + + *out_len = out_pos; + return result; +} diff --git a/zip/deflate64.h b/zip/deflate64.h new file mode 100644 index 0000000..f87df59 --- /dev/null +++ b/zip/deflate64.h @@ -0,0 +1,39 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef AARU_COMPRESSION_NATIVE_ZIP_DEFLATE64_H +#define AARU_COMPRESSION_NATIVE_ZIP_DEFLATE64_H + +#include +#include + +/** + * Decompress Deflate64 (ZIP method 9) data. + * + * Deflate64 is an extension of Deflate with a 64KB window, + * additional distance codes 30-31, and code 285 meaning 3 + 16 extra bits. + * + * @param in_buf Compressed input buffer (raw deflate64 stream, no zlib/gzip header) + * @param in_len Length of compressed input + * @param out_buf Decompressed output buffer (must be pre-allocated) + * @param out_len On entry: size of output buffer. On exit: bytes actually written. + * @return 0 on success, non-zero on error + */ +int zip_deflate64_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); + +#endif /* AARU_COMPRESSION_NATIVE_ZIP_DEFLATE64_H */ diff --git a/zip/implode.c b/zip/implode.c new file mode 100644 index 0000000..efc89c7 --- /dev/null +++ b/zip/implode.c @@ -0,0 +1,199 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "implode.h" + +#include +#include + +#include "../pak/bitstream.h" +#include "../pak/prefixcode.h" + +/* + * Parse a Shannon-Fano tree from the ZIP Implode bitstream. + * + * Format: 1 byte = number of groups - 1. + * Each group: 1 byte where high nibble = (count - 1), low nibble = (code length - 1). + * Codes are assigned from highest value down (shortest code = all ones). + */ +static PrefixCode *implode_read_tree(BitStream *bs, int num_symbols) +{ + int lengths[256]; + int num_groups; + int symbol; + int max_length; + + memset(lengths, 0, sizeof(lengths)); + + num_groups = (int)bitstream_read_bits_le(bs, 8) + 1; + + symbol = 0; + max_length = 0; + + for(int g = 0; g < num_groups; g++) + { + int group_byte = (int)bitstream_read_bits_le(bs, 8); + int count = ((group_byte >> 4) & 0x0F) + 1; + int length = (group_byte & 0x0F) + 1; + + for(int i = 0; i < count && symbol < num_symbols; i++) + { + lengths[symbol++] = length; + + if(length > max_length) max_length = length; + } + } + + /* Build prefix code tree. shortestCodeIsZeros=false means codes assigned from highest value down. */ + return prefix_code_alloc_with_lengths(lengths, symbol, max_length, false); +} + +int zip_implode_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, + int large_dictionary, int has_literals) +{ + BitStream bs; + PrefixCode *literal_code = NULL; + PrefixCode *length_code = NULL; + PrefixCode *distance_code = NULL; + int offset_bits; /* Number of low offset bits read raw */ + size_t out_pos = 0; + size_t out_size = *out_len; + int result = 0; + + if(!in_buf || !out_buf || !out_len) return -1; + + offset_bits = large_dictionary ? 7 : 6; + + bitstream_init(&bs, in_buf, in_len); + + /* Read trees */ + if(has_literals) + { + literal_code = implode_read_tree(&bs, 256); + + if(!literal_code) + { + result = -1; + goto cleanup; + } + } + + length_code = implode_read_tree(&bs, 64); + + if(!length_code) + { + result = -1; + goto cleanup; + } + + distance_code = implode_read_tree(&bs, 64); + + if(!distance_code) + { + result = -1; + goto cleanup; + } + + /* Decompress */ + while(out_pos < out_size && !bitstream_eof(&bs)) + { + uint32_t flag = bitstream_read_bits_le(&bs, 1); + + if(flag) /* Literal */ + { + uint8_t byte; + + if(has_literals) + { + int sym = prefix_code_read_symbol_le(&bs, literal_code); + + if(sym < 0) + { + result = -1; + goto cleanup; + } + + byte = (uint8_t)sym; + } + else + { + byte = (uint8_t)bitstream_read_bits_le(&bs, 8); + } + + out_buf[out_pos++] = byte; + } + else /* Match */ + { + /* Read low offset bits raw */ + uint32_t low_offset = bitstream_read_bits_le(&bs, offset_bits); + + /* Read high offset bits from distance tree */ + int high_offset = prefix_code_read_symbol_le(&bs, distance_code); + + if(high_offset < 0) + { + result = -1; + goto cleanup; + } + + size_t offset = ((size_t)high_offset << offset_bits) | low_offset; + offset += 1; /* Offset is 1-based */ + + /* Read length from tree */ + int length = prefix_code_read_symbol_le(&bs, length_code); + + if(length < 0) + { + result = -1; + goto cleanup; + } + + length += 2; /* Minimum match length is 2 */ + + /* If length value was 63 (max), read additional byte */ + if(length == 65) + { + int extra = (int)bitstream_read_bits_le(&bs, 8); + length += extra; + } + + /* If literal tree present, add 1 to length */ + if(has_literals) length++; + + /* Copy from history */ + for(int i = 0; i < length && out_pos < out_size; i++) + { + if(offset > out_pos) + out_buf[out_pos] = 0; + else + out_buf[out_pos] = out_buf[out_pos - offset]; + + out_pos++; + } + } + } + + *out_len = out_pos; + +cleanup: + if(literal_code) prefix_code_free(literal_code); + if(length_code) prefix_code_free(length_code); + if(distance_code) prefix_code_free(distance_code); + + return result; +} diff --git a/zip/implode.h b/zip/implode.h new file mode 100644 index 0000000..ac6424f --- /dev/null +++ b/zip/implode.h @@ -0,0 +1,41 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef AARU_COMPRESSION_NATIVE_ZIP_IMPLODE_H +#define AARU_COMPRESSION_NATIVE_ZIP_IMPLODE_H + +#include +#include + +/** + * Decompress ZIP Implode (method 6) data. + * + * ZIP Implode uses Shannon-Fano coded LZSS with LSB-first bitstream. + * + * @param in_buf Compressed input buffer + * @param in_len Length of compressed input + * @param out_buf Decompressed output buffer (must be pre-allocated) + * @param out_len On entry: size of output buffer. On exit: bytes actually written. + * @param large_dictionary Non-zero for 8K sliding dictionary (flag bit 1), zero for 4K + * @param has_literals Non-zero if literal tree is present (flag bit 2) + * @return 0 on success, non-zero on error + */ +int zip_implode_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, + int large_dictionary, int has_literals); + +#endif /* AARU_COMPRESSION_NATIVE_ZIP_IMPLODE_H */ diff --git a/zip/reduce.c b/zip/reduce.c new file mode 100644 index 0000000..a3aad0e --- /dev/null +++ b/zip/reduce.c @@ -0,0 +1,161 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "reduce.h" + +#include "../pak/bitstream.h" + +/* DLE escape byte used by ZIP Reduce */ +#define REDUCE_DLE 0x90 + +/* Minimum number of bits needed to represent (n-1) */ +static int reduce_b_value(int n) +{ + if(n > 16) return 5; + if(n > 8) return 4; + if(n > 4) return 3; + if(n > 2) return 2; + if(n > 0) return 1; + return 0; +} + +int zip_reduce_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, int comp_factor) +{ + BitStream bs; + uint8_t follower_sets[256][32]; + uint8_t follower_count[256]; + size_t out_pos = 0; + size_t out_size = *out_len; + int v_len_bits; + int v_len_mask; + + if(!in_buf || !out_buf || !out_len) return -1; + if(comp_factor < 1 || comp_factor > 4) return -1; + + v_len_bits = 8 - comp_factor; + v_len_mask = (1 << v_len_bits) - 1; + + bitstream_init(&bs, in_buf, in_len); + + /* Read follower sets in reverse order (255 down to 0) */ + for(int i = 255; i >= 0; i--) + { + follower_count[i] = (uint8_t)bitstream_read_bits_le(&bs, 6); + + if(follower_count[i] > 32) + { + *out_len = 0; + return -1; + } + + for(int j = 0; j < follower_count[i]; j++) follower_sets[i][j] = (uint8_t)bitstream_read_bits_le(&bs, 8); + } + + /* Decode using follower sets + LZ77 state machine */ + uint8_t last_char = 0; + int state = 0; + uint8_t v_byte = 0; + size_t match_len = 0; + + while(out_pos < out_size && !bitstream_eof(&bs)) + { + /* Read next byte through follower sets */ + uint8_t c; + + if(follower_count[last_char] == 0) { c = (uint8_t)bitstream_read_bits_le(&bs, 8); } + else + { + uint32_t flag = bitstream_read_bits_le(&bs, 1); + + if(flag == 1) { c = (uint8_t)bitstream_read_bits_le(&bs, 8); } + else + { + int bw = reduce_b_value(follower_count[last_char]); + int idx = (int)bitstream_read_bits_le(&bs, bw); + + if(idx >= follower_count[last_char]) + { + *out_len = out_pos; + return -1; + } + + c = follower_sets[last_char][idx]; + } + } + + last_char = c; + + /* LZ77 state machine */ + switch(state) + { + case 0: + if(c != REDUCE_DLE) { out_buf[out_pos++] = c; } + else + { + state = 1; + } + break; + + case 1: + if(c != 0) + { + v_byte = c; + match_len = v_byte & v_len_mask; + + if(match_len == (size_t)v_len_mask) + state = 2; /* Need extra length byte */ + else + state = 3; /* Read distance byte next */ + } + else + { + /* Escaped DLE: output literal 0x90 */ + out_buf[out_pos++] = REDUCE_DLE; + state = 0; + } + break; + + case 2: + match_len += c; + state = 3; + break; + + case 3: + { + size_t dist = ((size_t)(v_byte >> v_len_bits)) * 256 + c + 1; + match_len += 3; + + for(size_t i = 0; i < match_len && out_pos < out_size; i++) + { + if(dist > out_pos) + out_buf[out_pos] = 0; /* Before start of output = zeros */ + else + out_buf[out_pos] = out_buf[out_pos - dist]; + + out_pos++; + } + + state = 0; + break; + } + } + } + + *out_len = out_pos; + return 0; +} diff --git a/zip/reduce.h b/zip/reduce.h new file mode 100644 index 0000000..c4235ba --- /dev/null +++ b/zip/reduce.h @@ -0,0 +1,40 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef AARU_COMPRESSION_NATIVE_ZIP_REDUCE_H +#define AARU_COMPRESSION_NATIVE_ZIP_REDUCE_H + +#include +#include + +/** + * Decompress ZIP Reduce (methods 2-5) data. + * + * ZIP Reduce uses a two-stage compression: probabilistic follower sets + * followed by LZ77 with DLE (0x90) escape codes. + * + * @param in_buf Compressed input buffer + * @param in_len Length of compressed input + * @param out_buf Decompressed output buffer (must be pre-allocated) + * @param out_len On entry: size of output buffer. On exit: bytes actually written. + * @param comp_factor Compression factor 1-4 (from ZIP method 2-5: factor = method - 1) + * @return 0 on success, non-zero on error + */ +int zip_reduce_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len, int comp_factor); + +#endif /* AARU_COMPRESSION_NATIVE_ZIP_REDUCE_H */ diff --git a/zip/shrink.c b/zip/shrink.c new file mode 100644 index 0000000..5721a98 --- /dev/null +++ b/zip/shrink.c @@ -0,0 +1,227 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "shrink.h" + +#include +#include + +#include "../pak/bitstream.h" + +/* LZW constants for ZIP Shrink */ +#define SHRINK_MAX_BITS 13 +#define SHRINK_INIT_BITS 9 +#define SHRINK_MAX_CODES 8192 /* 2^13 */ +#define SHRINK_CONTROL 256 /* Control code */ +#define SHRINK_FIRST_CODE 257 /* First user code */ +#define SHRINK_GROW_CODE 1 /* Increment code size */ +#define SHRINK_CLEAR_CODE 2 /* Partial clear */ + +/* LZW dictionary entry */ +typedef struct +{ + int16_t parent; /* Parent code, -1 for root entries */ + uint8_t chr; /* Character at this node */ + uint8_t used; /* Whether this entry is in use */ +} ShrinkEntry; + +/* Reverse-walk the dictionary chain and write to a stack buffer, return length */ +static int shrink_decode_string(ShrinkEntry *dict, int code, uint8_t *stack, int max_stack) +{ + int count = 0; + + while(code >= 0 && count < max_stack) + { + stack[count++] = dict[code].chr; + code = dict[code].parent; + } + + return count; +} + +/* Partial clear: mark entries whose parent chains include non-root entries + that are not themselves parents of other entries */ +static void shrink_partial_clear(ShrinkEntry *dict, int num_codes) +{ + /* Mark entries that are referenced as parents */ + uint8_t *is_parent = (uint8_t *)calloc(SHRINK_MAX_CODES, 1); + if(!is_parent) return; + + for(int i = SHRINK_FIRST_CODE; i < num_codes; i++) + { + if(dict[i].used && dict[i].parent >= SHRINK_FIRST_CODE) is_parent[dict[i].parent] = 1; + } + + /* Clear entries that are not parents of other entries */ + for(int i = SHRINK_FIRST_CODE; i < num_codes; i++) + { + if(dict[i].used && !is_parent[i]) dict[i].used = 0; + } + + free(is_parent); +} + +int zip_shrink_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len) +{ + BitStream bs; + ShrinkEntry *dict; + uint8_t stack[SHRINK_MAX_CODES]; + size_t out_pos = 0; + size_t out_size = *out_len; + int code_size; + int next_code; + int prev_code; + int code; + int len; + + if(!in_buf || !out_buf || !out_len) return -1; + + /* Allocate dictionary */ + dict = (ShrinkEntry *)calloc(SHRINK_MAX_CODES, sizeof(ShrinkEntry)); + if(!dict) return -1; + + /* Initialize dictionary with single-character entries */ + for(int i = 0; i < 256; i++) + { + dict[i].parent = -1; + dict[i].chr = (uint8_t)i; + dict[i].used = 1; + } + + /* Code 256 is the control code, mark as used */ + dict[SHRINK_CONTROL].parent = -1; + dict[SHRINK_CONTROL].chr = 0; + dict[SHRINK_CONTROL].used = 1; + + next_code = SHRINK_FIRST_CODE; + code_size = SHRINK_INIT_BITS; + + bitstream_init(&bs, in_buf, in_len); + + /* Read first code */ + prev_code = (int)bitstream_read_bits_le(&bs, code_size); + + if(prev_code < 256 && out_pos < out_size) out_buf[out_pos++] = (uint8_t)prev_code; + + while(out_pos < out_size && !bitstream_eof(&bs)) + { + code = (int)bitstream_read_bits_le(&bs, code_size); + + if(bitstream_eof(&bs)) break; + + /* Handle control code */ + if(code == SHRINK_CONTROL) + { + int subcode = (int)bitstream_read_bits_le(&bs, code_size); + + if(bitstream_eof(&bs)) break; + + if(subcode == SHRINK_GROW_CODE) + { + code_size++; + if(code_size > SHRINK_MAX_BITS) + { + free(dict); + return -1; + } + } + else if(subcode == SHRINK_CLEAR_CODE) + { + shrink_partial_clear(dict, next_code); + + /* Reset next_code to search from the beginning for freed entries */ + next_code = SHRINK_FIRST_CODE; + while(next_code < SHRINK_MAX_CODES && dict[next_code].used) next_code++; + } + + continue; + } + + /* Handle KwKwK case: code == next available and not yet in dictionary */ + if(code >= SHRINK_FIRST_CODE && !dict[code].used) + { + /* The code is the next one to be added: reconstruct prev_code string + first char */ + len = shrink_decode_string(dict, prev_code, stack, SHRINK_MAX_CODES); + + if(len <= 0) + { + free(dict); + return -1; + } + + /* First char of previous string is stack[len-1] (stack is in reverse order) */ + uint8_t first_char = stack[len - 1]; + + /* Output in forward order: stack is reversed */ + for(int i = len - 1; i >= 0; i--) + { + if(out_pos < out_size) out_buf[out_pos++] = stack[i]; + } + + if(out_pos < out_size) out_buf[out_pos++] = first_char; + + /* Add new dictionary entry */ + if(next_code < SHRINK_MAX_CODES) + { + dict[next_code].parent = (int16_t)prev_code; + dict[next_code].chr = first_char; + dict[next_code].used = 1; + + /* Find next free slot */ + next_code++; + while(next_code < SHRINK_MAX_CODES && dict[next_code].used) next_code++; + } + + prev_code = code; + continue; + } + + /* Normal case: decode the string */ + len = shrink_decode_string(dict, code, stack, SHRINK_MAX_CODES); + + if(len <= 0) + { + free(dict); + return -1; + } + + /* Output in forward order (stack is reversed) */ + for(int i = len - 1; i >= 0; i--) + { + if(out_pos < out_size) out_buf[out_pos++] = stack[i]; + } + + /* Add new dictionary entry: prev_code + first char of current string */ + if(next_code < SHRINK_MAX_CODES) + { + dict[next_code].parent = (int16_t)prev_code; + dict[next_code].chr = stack[len - 1]; /* first char of decoded string */ + dict[next_code].used = 1; + + /* Find next free slot (partial clear may have freed interior slots) */ + next_code++; + while(next_code < SHRINK_MAX_CODES && dict[next_code].used) next_code++; + } + + prev_code = code; + } + + free(dict); + *out_len = out_pos; + return 0; +} diff --git a/zip/shrink.h b/zip/shrink.h new file mode 100644 index 0000000..91c34b7 --- /dev/null +++ b/zip/shrink.h @@ -0,0 +1,39 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef AARU_COMPRESSION_NATIVE_ZIP_SHRINK_H +#define AARU_COMPRESSION_NATIVE_ZIP_SHRINK_H + +#include +#include + +/** + * Decompress ZIP Shrink (method 1) data. + * + * ZIP Shrink uses a modified LZW algorithm with variable code sizes (9-13 bits), + * LSB-first bit reading, and control code 256 for code width growth and partial clear. + * + * @param in_buf Compressed input buffer + * @param in_len Length of compressed input + * @param out_buf Decompressed output buffer (must be pre-allocated) + * @param out_len On entry: size of output buffer. On exit: bytes actually written. + * @return 0 on success, non-zero on error + */ +int zip_shrink_decompress(const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t *out_len); + +#endif /* AARU_COMPRESSION_NATIVE_ZIP_SHRINK_H */ diff --git a/zip/zip.c b/zip/zip.c new file mode 100644 index 0000000..37a7f51 --- /dev/null +++ b/zip/zip.c @@ -0,0 +1,335 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "zip.h" + +#include +#include + +/* PPMd includes */ +#include "../ppmd/SubAllocatorVariantI.h" +#include "../ppmd/VariantI.h" + +/* WavPack includes */ +#include "../wavpack/wavpack.h" + +/* WinZipJPEG includes */ +#include "../winzipjpeg/Decompressor.h" + +/* ============== PPMd Wrapper ============== */ + +typedef struct +{ + const uint8_t *data; + size_t size; + size_t pos; +} PPMdBufferContext; + +static int ppmd_read_byte(void *context) +{ + PPMdBufferContext *ctx = (PPMdBufferContext *)context; + + if(ctx->pos >= ctx->size) return -1; + + return ctx->data[ctx->pos++]; +} + +int zip_ppmd_decompress(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, size_t src_size, int max_order, + int sub_alloc_size, int restoration) +{ + PPMdSubAllocatorVariantI *alloc; + PPMdModelVariantI model; + PPMdBufferContext ctx; + + if(!dst_buffer || !src_buffer) return -1; + + ctx.data = src_buffer; + ctx.size = src_size; + ctx.pos = 0; + + alloc = CreateSubAllocatorVariantI(sub_alloc_size); + + if(!alloc) return -1; + + StartPPMdModelVariantI(&model, ppmd_read_byte, &ctx, alloc, max_order, restoration); + + for(size_t i = 0; i < dst_size; i++) + { + int byte = NextPPMdVariantIByte(&model); + + if(byte < 0) + { + FreeSubAllocatorVariantI(alloc); + return -1; + } + + dst_buffer[i] = (uint8_t)byte; + } + + FreeSubAllocatorVariantI(alloc); + return 0; +} + +/* ============== WavPack Wrapper ============== */ + +typedef struct +{ + const uint8_t *data; + size_t size; + size_t pos; + int pushback; + int has_pushback; +} WavpackBufferContext; + +static int32_t wavpack_read_bytes(void *id, void *data, int32_t bcount) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + int32_t available = (int32_t)(ctx->size - ctx->pos); + int32_t to_read = bcount < available ? bcount : available; + + if(ctx->has_pushback && to_read > 0) + { + ((uint8_t *)data)[0] = (uint8_t)ctx->pushback; + ctx->has_pushback = 0; + memcpy((uint8_t *)data + 1, ctx->data + ctx->pos, to_read - 1); + ctx->pos += to_read - 1; + } + else + { + memcpy(data, ctx->data + ctx->pos, to_read); + ctx->pos += to_read; + } + + return to_read; +} + +static uint32_t wavpack_get_pos(void *id) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + return (uint32_t)ctx->pos; +} + +static int wavpack_set_pos_abs(void *id, uint32_t pos) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + + if(pos > ctx->size) return -1; + + ctx->pos = pos; + ctx->has_pushback = 0; + return 0; +} + +static int wavpack_set_pos_rel(void *id, int32_t delta, int mode) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + int64_t newpos; + + switch(mode) + { + case 0: /* SEEK_SET */ + newpos = delta; + break; + case 1: /* SEEK_CUR */ + newpos = (int64_t)ctx->pos + delta; + break; + case 2: /* SEEK_END */ + newpos = (int64_t)ctx->size + delta; + break; + default: + return -1; + } + + if(newpos < 0 || (size_t)newpos > ctx->size) return -1; + + ctx->pos = (size_t)newpos; + ctx->has_pushback = 0; + return 0; +} + +static int wavpack_push_back_byte(void *id, int c) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + ctx->pushback = c; + ctx->has_pushback = 1; + return c; +} + +static uint32_t wavpack_get_length(void *id) +{ + WavpackBufferContext *ctx = (WavpackBufferContext *)id; + return (uint32_t)ctx->size; +} + +static int wavpack_can_seek(void *id) +{ + (void)id; + return 1; +} + +int zip_wavpack_decompress(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t src_size, + uint32_t num_samples, int bits_per_sample, int num_channels) +{ + WavpackBufferContext ctx; + WavpackStreamReader reader; + WavpackContext *wpc; + char error[80]; + int bytes_per_sample; + int32_t *sample_buf; + uint32_t decoded; + size_t out_pos; + + if(!dst_buffer || !dst_size || !src_buffer) return -1; + + ctx.data = src_buffer; + ctx.size = src_size; + ctx.pos = 0; + ctx.pushback = 0; + ctx.has_pushback = 0; + + reader.read_bytes = wavpack_read_bytes; + reader.get_pos = wavpack_get_pos; + reader.set_pos_abs = wavpack_set_pos_abs; + reader.set_pos_rel = wavpack_set_pos_rel; + reader.push_back_byte = wavpack_push_back_byte; + reader.get_length = wavpack_get_length; + reader.can_seek = wavpack_can_seek; + reader.write_bytes = NULL; + + wpc = WavpackOpenFileInputEx(&reader, &ctx, NULL, error, 0, 0); + + if(!wpc) return -1; + + bytes_per_sample = (bits_per_sample + 7) / 8; + + /* Allocate sample buffer for one block of samples */ + sample_buf = (int32_t *)malloc(num_samples * num_channels * sizeof(int32_t)); + + if(!sample_buf) + { + WavpackCloseFile(wpc); + return -1; + } + + decoded = WavpackUnpackSamples(wpc, sample_buf, num_samples); + out_pos = 0; + + /* Compact int32 samples to actual byte width */ + for(uint32_t i = 0; i < decoded * (uint32_t)num_channels; i++) + { + int32_t sample = sample_buf[i]; + + for(int b = 0; b < bytes_per_sample; b++) + { + if(out_pos < *dst_size) dst_buffer[out_pos++] = (uint8_t)(sample & 0xFF); + + sample >>= 8; + } + } + + *dst_size = out_pos; + + free(sample_buf); + WavpackCloseFile(wpc); + return 0; +} + +/* ============== WinZipJPEG Wrapper ============== */ + +typedef struct +{ + const uint8_t *data; + size_t size; + size_t pos; +} WinZipJPEGBufferContext; + +static size_t winzipjpeg_read(void *context, uint8_t *buffer, size_t length) +{ + WinZipJPEGBufferContext *ctx = (WinZipJPEGBufferContext *)context; + size_t remaining = ctx->size - ctx->pos; + size_t to_read = length < remaining ? length : remaining; + + memcpy(buffer, ctx->data + ctx->pos, to_read); + ctx->pos += to_read; + + return to_read; +} + +int zip_winzipjpeg_decompress(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t src_size) +{ + WinZipJPEGBufferContext ctx; + WinZipJPEGDecompressor *decompressor; + size_t out_pos = 0; + size_t out_size = *dst_size; + int err; + + if(!dst_buffer || !dst_size || !src_buffer) return -1; + + ctx.data = src_buffer; + ctx.size = src_size; + ctx.pos = 0; + + decompressor = AllocWinZipJPEGDecompressor(winzipjpeg_read, &ctx); + + if(!decompressor) return -1; + + err = ReadWinZipJPEGHeader(decompressor); + + if(err != WinZipJPEGNoError) + { + FreeWinZipJPEGDecompressor(decompressor); + return err; + } + + /* Process bundles */ + while(!IsFinalWinZipJPEGBundle(decompressor)) + { + err = ReadNextWinZipJPEGBundle(decompressor); + + if(err != WinZipJPEGNoError) break; + + /* Copy metadata bytes */ + uint32_t meta_len = WinZipJPEGBundleMetadataLength(decompressor); + uint8_t *meta = WinZipJPEGBundleMetadataBytes(decompressor); + + for(uint32_t i = 0; i < meta_len && out_pos < out_size; i++) dst_buffer[out_pos++] = meta[i]; + + /* Process slices */ + while(AreMoreWinZipJPEGSlicesAvailable(decompressor)) + { + err = ReadNextWinZipJPEGSlice(decompressor); + + if(err != WinZipJPEGNoError) break; + + /* Encode blocks to output buffer */ + while(AreMoreWinZipJPEGBytesAvailable(decompressor) && out_pos < out_size) + { + size_t written = EncodeWinZipJPEGBlocksToBuffer(decompressor, dst_buffer + out_pos, out_size - out_pos); + out_pos += written; + + if(written == 0) break; + } + } + + if(err != WinZipJPEGNoError) break; + } + + *dst_size = out_pos; + FreeWinZipJPEGDecompressor(decompressor); + return (err == WinZipJPEGNoError) ? 0 : err; +} diff --git a/zip/zip.h b/zip/zip.h new file mode 100644 index 0000000..23c090e --- /dev/null +++ b/zip/zip.h @@ -0,0 +1,71 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2026 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef AARU_COMPRESSION_NATIVE_ZIP_ZIP_H +#define AARU_COMPRESSION_NATIVE_ZIP_ZIP_H + +#include +#include + +#include "deflate64.h" +#include "implode.h" +#include "reduce.h" +#include "shrink.h" + +/** + * Decompress ZIP PPMd (method 98, variant I) data. + * + * @param dst_buffer Output buffer + * @param dst_size Size of output buffer / bytes to decompress + * @param src_buffer Compressed input buffer + * @param src_size Size of compressed input + * @param max_order PPMd model order (1-16) + * @param sub_alloc_size Sub-allocator memory size in bytes + * @param restoration Model restoration method (0=restart, 1=cutoff, 2=freeze) + * @return 0 on success, non-zero on error + */ +int zip_ppmd_decompress(uint8_t *dst_buffer, size_t dst_size, const uint8_t *src_buffer, size_t src_size, int max_order, + int sub_alloc_size, int restoration); + +/** + * Decompress WinZip WavPack (ZIP method 97) data. + * + * @param dst_buffer Output buffer for raw audio samples + * @param dst_size On entry: size of output buffer. On exit: bytes written. + * @param src_buffer Compressed WavPack data + * @param src_size Size of compressed input + * @param num_samples Number of audio samples to decode + * @param bits_per_sample Bits per sample (8, 16, 24, 32) + * @param num_channels Number of audio channels + * @return 0 on success, non-zero on error + */ +int zip_wavpack_decompress(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t src_size, + uint32_t num_samples, int bits_per_sample, int num_channels); + +/** + * Decompress WinZip JPEG (ZIP method 96) data. + * + * @param dst_buffer Output buffer for reconstructed JPEG data + * @param dst_size On entry: size of output buffer. On exit: bytes written. + * @param src_buffer Compressed WinZipJPEG data + * @param src_size Size of compressed input + * @return 0 on success, non-zero on error + */ +int zip_winzipjpeg_decompress(uint8_t *dst_buffer, size_t *dst_size, const uint8_t *src_buffer, size_t src_size); + +#endif /* AARU_COMPRESSION_NATIVE_ZIP_ZIP_H */