claunia/cuetools.net
1
0
Fork 0
mirror of https://github.com/claunia/cuetools.net.git synced 2025-12-16 18:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

FLACCL: more precise upper limit on coef size, allowing for better precision,

Browse Source 
while still being able to do all math in 32 bit integers.
This commit is contained in:
Grigory Chudov
2014-05-11 18:56:29 -04:00
parent 5c9a6c186b
commit 412a76fb42
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
CUETools.Codecs.FLACCL/flac.cl
View File

@@ -796,7 +796,14 @@ void clQuantizeLPC(
int cbits = min(51 - 2 * clz(shared.task.blocksize), shared.task.abits) - minprecision + (i - ((i >> precisions) << precisions));
#if BITS_PER_SAMPLE <= 16
// Limit cbits so that 32-bit arithmetics will be enough when calculating residual
cbits = min(cbits, clz(order + 1) + 1 - shared.task.obits);
// (1 << (obits - 1)) * ((1 << (cbits - 1)) - 1) * (order + 1) < (1 << 31)
// (1 << (cbits - 1)) - 1 < (1 << (32 - obits)) / (order + 1)
// (1 << (cbits - 1)) <= (1 << (32 - obits)) / (order + 1)
// (1 << (cbits - 1)) <= (1 << (32 - obits - (32 - clz(order))) <= (1 << (32 - obits)) / (order + 1)
// (1 << (cbits - 1)) <= (1 << (clz(order) - obits))
// cbits - 1 <= clz(order) - obits
// cbits <= clz(order) - obits + 1
cbits = min(cbits, clz(order) + 1 - shared.task.obits);
#endif
cbits = clamp(cbits, 3, 15);