claunia/cuetools.net
1
0
Fork 0
mirror of https://github.com/claunia/cuetools.net.git synced 2025-12-16 18:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

CTDB: fast parity/syndrome calculation

Browse Source 
CTDB: initial submission without a parity file
CTDB: track CRCs submission
This commit is contained in:
chudov
2011-04-24 21:07:50 +00:00
parent 153633c991
commit 1cc474e170
10 changed files with 400 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CUERipper/frmCUERipper.cs
View File

@@ -390,7 +390,6 @@ namespace CUERipper
else
cueSheet.CTDB.Submit(
(int)cueSheet.ArVerify.WorstConfidence() + 1,
(int)cueSheet.ArVerify.WorstTotal() + 1,
cueSheet.Artist,
cueSheet.Title);
}

155
CUETools.AccurateRip/AccurateRip.cs
View File

@@ -434,18 +434,53 @@ namespace CUETools.AccurateRip
_CRCLOG[iTrack] = value;
}
internal ushort[,] syndrome;
public unsafe ushort[,] GetSyndrome()
{
if (!calcParity)
throw new InvalidOperationException();
var syndrome = new ushort[stride, npar];
fixed (byte* pbpar = parity)
fixed (ushort* psyn0 = syndrome, plog = Galois16.instance.LogTbl, pexp = Galois16.instance.ExpTbl)
{
ushort* ppar = (ushort*)pbpar;
for (int y = 0; y < stride; y++)
{
ushort* syn = psyn0 + y * npar;
for (int x1 = 0; x1 < npar; x1++)
{
ushort lo = ppar[y * npar + x1];
if (lo != 0)
{
var llo = plog[lo] + 0xffff;
for (int x = 0; x < npar; x++)
syn[x] ^= pexp[llo - (1 + x1) * x];
}
}
}
}
return syndrome;
}
public unsafe ushort[,] Syndrome
{
get
{
if (syndrome == null)
syndrome = GetSyndrome();
return syndrome;
}
}
private ushort[,] syndrome;
internal byte[] parity;
internal ushort[, ,] encodeTable;
internal ushort[, ,] decodeTable;
private int maxOffset;
internal ushort[] leadin;
internal ushort[] leadout;
private int stride = 1, laststride = 1, stridecount = 1, npar = 1;
private bool calcSyn = false;
private bool calcParity = false;
internal void InitCDRepair(int stride, int laststride, int stridecount, int npar, bool calcSyn, bool calcParity)
internal void InitCDRepair(int stride, int laststride, int stridecount, int npar, bool calcParity)
{
if (npar != 8)
throw new ArgumentOutOfRangeException("npar");
@@ -455,7 +490,6 @@ namespace CUETools.AccurateRip
this.laststride = laststride;
this.stridecount = stridecount;
this.npar = npar;
this.calcSyn = calcSyn;
this.calcParity = calcParity;
Init(_toc);
}
@@ -512,25 +546,14 @@ namespace CUETools.AccurateRip
return CTDBCRC(0, offset, stride / 2, laststride / 2);
}
private unsafe static void CalcSyn8(ushort* pt, ushort* syn, uint lo, uint hi)
{
syn[0] ^= (ushort)lo;
syn[1] = (ushort)(lo ^ pt[(syn[1] & 255) * 16 + 1] ^ pt[(syn[1] >> 8) * 16 + 1 + 8]);
syn[2] = (ushort)(lo ^ pt[(syn[2] & 255) * 16 + 2] ^ pt[(syn[2] >> 8) * 16 + 2 + 8]);
syn[3] = (ushort)(lo ^ pt[(syn[3] & 255) * 16 + 3] ^ pt[(syn[3] >> 8) * 16 + 3 + 8]);
syn[4] = (ushort)(lo ^ pt[(syn[4] & 255) * 16 + 4] ^ pt[(syn[4] >> 8) * 16 + 4 + 8]);
syn[5] = (ushort)(lo ^ pt[(syn[5] & 255) * 16 + 5] ^ pt[(syn[5] >> 8) * 16 + 5 + 8]);
syn[6] = (ushort)(lo ^ pt[(syn[6] & 255) * 16 + 6] ^ pt[(syn[6] >> 8) * 16 + 6 + 8]);
syn[7] = (ushort)(lo ^ pt[(syn[7] & 255) * 16 + 7] ^ pt[(syn[7] >> 8) * 16 + 7 + 8]);
syn[8] ^= (ushort)hi;
syn[9] = (ushort)(hi ^ pt[(syn[9] & 255) * 16 + 1] ^ pt[(syn[9] >> 8) * 16 + 1 + 8]);
syn[10] = (ushort)(hi ^ pt[(syn[10] & 255) * 16 + 2] ^ pt[(syn[10] >> 8) * 16 + 2 + 8]);
syn[11] = (ushort)(hi ^ pt[(syn[11] & 255) * 16 + 3] ^ pt[(syn[11] >> 8) * 16 + 3 + 8]);
syn[12] = (ushort)(hi ^ pt[(syn[12] & 255) * 16 + 4] ^ pt[(syn[12] >> 8) * 16 + 4 + 8]);
syn[13] = (ushort)(hi ^ pt[(syn[13] & 255) * 16 + 5] ^ pt[(syn[13] >> 8) * 16 + 5 + 8]);
syn[14] = (ushort)(hi ^ pt[(syn[14] & 255) * 16 + 6] ^ pt[(syn[14] >> 8) * 16 + 6 + 8]);
syn[15] = (ushort)(hi ^ pt[(syn[15] & 255) * 16 + 7] ^ pt[(syn[15] >> 8) * 16 + 7 + 8]);
}
//private unsafe static void CalcPar8(ushort* pt, ushort* wr, uint lo)
//{
// uint wrlo = wr[0] ^ lo;
// ushort* ptiblo0 = pt + (wrlo & 255) * 16;
// ushort* ptiblo1 = pt + (wrlo >> 8) * 16 + 8;
// ((ulong*)wr)[0] = ((ulong*)(wr + 1))[0] ^ ((ulong*)ptiblo0)[0] ^ ((ulong*)ptiblo1)[0];
// ((ulong*)wr)[1] = (((ulong*)(wr))[1] >> 16) ^ ((ulong*)ptiblo0)[1] ^ ((ulong*)ptiblo1)[1];
//}
private unsafe static void CalcPar8(ushort* pt, ushort* wr, uint lo, uint hi)
{
@@ -548,15 +571,22 @@ namespace CUETools.AccurateRip
((ulong*)wr)[3] = (((ulong*)(wr))[3] >> 16) ^ ((ulong*)ptibhi0)[1] ^ ((ulong*)ptibhi1)[1];
#else
const int npar = 8;
ushort* ptiblo = pt + (wr[0] ^ lo) * npar;
ushort* ptibhi = pt + (wr[npar] ^ hi) * npar;
uint s = wr[0] ^ lo;
ushort* ptib0 = pt + (s & 255) * 16;
ushort* ptib1 = pt + (s >> 8) * 16 + 8;
for (int i = 0; i < npar - 1; i++)
{
wr[i] = (ushort)(wr[i + 1] ^ ptiblo[i]);
wr[npar + i] = (ushort)(wr[i + 1 + npar] ^ ptibhi[i]);
}
wr[npar - 1] = ptiblo[npar - 1];
wr[2 * npar - 1] = ptibhi[npar - 1];
wr[i] = (ushort)(wr[i + 1] ^ ptib0[i] ^ ptib1[i]);
wr[npar - 1] = (ushort)(ptib0[npar - 1] ^ ptib1[npar - 1]);
wr += 8;
s = wr[0] ^ hi;
ptib0 = pt + (s & 255) * 16;
ptib1 = pt + (s >> 8) * 16 + 8;
for (int i = 0; i < npar - 1; i++)
wr[i] = (ushort)(wr[i + 1] ^ ptib0[i] ^ ptib1[i]);
wr[npar - 1] = (ushort)(ptib0[npar - 1] ^ ptib1[npar - 1]);
#endif
}
@@ -575,12 +605,10 @@ namespace CUETools.AccurateRip
/// <param name="pSampleBuff"></param>
/// <param name="count"></param>
/// <param name="offs"></param>
public unsafe void CalculateCRCs(uint* t, ushort* syn, ushort* wr, ushort* pte, ushort* ptd, uint* pSampleBuff, int count, int offs)
public unsafe void CalculateCRCs(uint* t, ushort* wr, ushort* pte, uint* pSampleBuff, int count, int offs)
{
int currentStride = ((int)_sampleCount * 2) / stride;
bool doSyn = currentStride >= 1 && currentStride <= stridecount && calcSyn;
bool doPar = currentStride >= 1 && currentStride <= stridecount && calcParity;
uint n = (uint)(stridecount - currentStride);
int crcTrack = _currentTrack + (_samplesDoneTrack == 0 && _currentTrack > 0 ? -1 : 0);
uint crcar = _CRCAR[_currentTrack, 0];
@@ -620,9 +648,9 @@ namespace CUETools.AccurateRip
}
uint hi = sample >> 16;
if (doSyn) CalcSyn8(ptd, syn + i * 16, lo, hi);
if (doPar) CalcPar8(pte, wr + i * 16, lo, hi);
//if (doPar) CalcPar8(pte, wr + i * 16, lo);
//uint hi = sample >> 16;
crc32 = (crc32 >> 8) ^ t[(byte)(crc32 ^ hi)];
crc32 = (crc32 >> 8) ^ t[(byte)(crc32 ^ (hi >> 8))];
@@ -632,6 +660,7 @@ namespace CUETools.AccurateRip
crcwn = (crcwn >> 8) ^ t[(byte)(crcwn ^ (hi >> 8))];
crcnl++;
}
//if (doPar) CalcPar8(pte, wr + i * 16 + 8, hi);
int pk = ((int)(lo << 16)) >> 16;
peak = Math.Max(peak, (pk << 1) ^ (pk >> 31));
@@ -683,7 +712,7 @@ namespace CUETools.AccurateRip
int pos = 0;
fixed (uint* t = Crc32.table)
fixed (ushort* synptr1 = syndrome, pte = encodeTable, ptd = decodeTable)
fixed (ushort* pte = encodeTable)
fixed (byte* pSampleBuff = &sampleBuffer.Bytes[0], bpar = parity)
while (pos < sampleBuffer.Length)
{
@@ -691,7 +720,7 @@ namespace CUETools.AccurateRip
int copyCount = Math.Min(Math.Min(sampleBuffer.Length - pos, (int)_samplesRemTrack), 588 - (int)_sampleCount % 588);
uint* samples = ((uint*)pSampleBuff) + pos;
int currentPart = ((int)_sampleCount * 2) % stride;
ushort* synptr = synptr1 + npar * currentPart;
//ushort* synptr = synptr1 + npar * currentPart;
ushort* wr = ((ushort*)bpar) + npar * currentPart;
int currentStride = ((int)_sampleCount * 2) / stride;
@@ -711,7 +740,7 @@ namespace CUETools.AccurateRip
: _samplesDoneTrack >= 445 * 588 && _samplesDoneTrack <= 455 * 588 ? 2 * maxOffset + 1 + _samplesDoneTrack - 445 * 588
: -1;
CalculateCRCs(t, synptr, wr, pte, ptd, samples, copyCount, offset);
CalculateCRCs(t, wr, pte, samples, copyCount, offset);
// duplicate prefix to suffix
if (_samplesDoneTrack < maxOffset && _samplesRemTrack <= maxOffset)
@@ -755,7 +784,7 @@ namespace CUETools.AccurateRip
}
}
public void Combine(AccurateRipVerify part, int start, int end)
public unsafe void Combine(AccurateRipVerify part, int start, int end)
{
for (int i = 0; i < leadin.Length; i++)
{
@@ -827,6 +856,33 @@ namespace CUETools.AccurateRip
}
_Peak[iTrack] = Math.Max(_Peak[iTrack], part._Peak[iTrack]);
}
if (calcParity)
{
var newSyndrome1 = this.GetSyndrome();
var newSyndrome2 = part.GetSyndrome();
var i1 = Math.Max(0, start * 2 - stride);
var i2 = Math.Min(2 * (int)_finalSampleCount - laststride - stride, end * 2 - stride);
var diff = i2 / stride - i1 / stride;
var i1s = i1 % stride;
var i2s = i2 % stride;
var imin = Math.Min(i1s, i2s);
var imax = Math.Max(i1s, i2s);
var diff1 = diff + (imin < i2s ? 1 : 0) - (imin < i1s ? 1 : 0);
for (int i = 0; i < stride; i++)
for (int j = 0; j < npar; j++)
{
var d1 = j * (i >= imin && i < imax ? diff1 : diff);
newSyndrome1[i, j] = (ushort)(newSyndrome2[i, j] ^ Galois16.instance.mulExp(newSyndrome1[i, j], (d1 & 0xffff) + (d1 >> 16)));
}
fixed (byte* p = this.parity)
fixed (ushort* syn = newSyndrome1)
{
ushort* p1 = (ushort*)p;
for (int i = 0; i < stride; i++)
Galois16.instance.Syndrome2Parity(syn + i * npar, p1 + i * npar, npar);
}
}
}
public void Init(CDImageLayout toc)
@@ -838,7 +894,7 @@ namespace CUETools.AccurateRip
for (int iTrack = 1; iTrack <= _toc.AudioTracks; iTrack++)
_CRCMASK[iTrack] = 0xffffffff ^ Crc32.Combine(0xffffffff, 0, (int)_toc[iTrack + _toc.FirstAudio - 1].Length * 588 * 4);
maxOffset = Math.Max(4096 * 2, (calcSyn || calcParity) ? stride + laststride : 0);
maxOffset = Math.Max(4096 * 2, calcParity ? stride + laststride : 0);
if (maxOffset % 588 != 0)
maxOffset += 588 - maxOffset % 588;
_CRCAR = new uint[_toc.AudioTracks + 1, 3 * maxOffset];
@@ -851,15 +907,13 @@ namespace CUETools.AccurateRip
_CRCV2 = new uint[_toc.AudioTracks + 1, 3 * maxOffset];
_Peak = new int[_toc.AudioTracks + 1];
syndrome = new ushort[calcSyn ? stride : 1, npar];
parity = new byte[stride * npar * 2];
syndrome = null;
parity = calcParity ? new byte[stride * npar * 2] : null;
if (calcParity && npar == 8)
encodeTable = Galois16.instance.makeEncodeTable(npar);
if (calcSyn && npar == 8)
decodeTable = Galois16.instance.makeDecodeTable(npar);
int leadin_len = Math.Max(4096 * 4, (calcSyn || calcParity) ? stride * 2 : 0);
int leadout_len = Math.Max(4096 * 4, (calcSyn || calcParity) ? stride + laststride : 0);
int leadin_len = Math.Max(4096 * 4, calcParity ? stride * 2 : 0);
int leadout_len = Math.Max(4096 * 4, calcParity ? stride + laststride : 0);
leadin = new ushort[leadin_len];
leadout = new ushort[leadout_len];
_currentTrack = 0;
@@ -1025,6 +1079,11 @@ namespace CUETools.AccurateRip
get { return AudioPCMConfig.RedBook; }
}
public CDImageLayout TOC
{
get { return _toc; }
}
public long FinalSampleCount
{
get

50
CUETools.AccurateRip/CDRepair.cs
View File

@@ -57,6 +57,10 @@ namespace CUETools.AccurateRip
public long FinalSampleCount
{
get
{
return finalSampleCount;
}
set
{
if (value < 0) // != _toc.Length?
@@ -72,22 +76,25 @@ namespace CUETools.AccurateRip
return npar;
}
}
public int Stride
{
get
{
return stride;
}
}
}
public class CDRepairEncode : CDRepair
{
internal bool verify;
internal bool encode;
protected AccurateRipVerify ar;
public CDRepairEncode(AccurateRipVerify ar, int stride, int npar, bool verify, bool encode)
public CDRepairEncode(AccurateRipVerify ar, int stride, int npar)
: base ((int)ar.FinalSampleCount, stride, npar)
{
this.ar = ar;
this.verify = verify;
this.encode = encode;
ar.InitCDRepair(stride, laststride, stridecount, npar, verify, encode);
ar.InitCDRepair(stride, laststride, stridecount, npar, true);
}
//private unsafe void ProcessStride(int currentStride, int currentPart, int count, ushort* data)
@@ -265,6 +272,14 @@ namespace CUETools.AccurateRip
return VerifyParity(npar, parity2, 0, parity2.Length, actualOffset);
}
public AccurateRipVerify AR
{
get
{
return ar;
}
}
public uint CRC
{
get
@@ -273,12 +288,21 @@ namespace CUETools.AccurateRip
}
}
public string TrackCRCs
{
get
{
var sb = new StringBuilder();
for (int i = 1; i <= ar.TOC.AudioTracks; i++)
sb.AppendFormat(" {0:x8}", ar.CTDBCRC(i, 0, stride / 2, laststride / 2));
return sb.ToString().Substring(1);
}
}
public unsafe bool FindOffset(int npar2, byte[] parity2, int pos, uint expectedCRC, out int actualOffset, out bool hasErrors)
{
if (npar2 != npar)
throw new Exception("npar mismatch");
if (!verify)
throw new Exception("verify was not enabled");
if (ar.Position != ar.FinalSampleCount)
throw new Exception("ar.Position != ar.FinalSampleCount");
@@ -290,6 +314,7 @@ namespace CUETools.AccurateRip
int* _errpos = stackalloc int[npar];
int* syn = stackalloc int[npar];
bool foundOffset = false;
var arSyndrome = ar.Syndrome;
for (int allowed_errors = 0; allowed_errors < npar / 2 && !foundOffset; allowed_errors++)
{
@@ -307,7 +332,7 @@ namespace CUETools.AccurateRip
for (int i = 0; i < npar; i++)
{
int synI = ar.syndrome[part, i];
int synI = arSyndrome[part, i];
// offset < 0
if (part < -offset * 2)
@@ -365,6 +390,7 @@ namespace CUETools.AccurateRip
{
int* syn = stackalloc int[npar];
int offset = fix.actualOffset;
var arSyndrome = ar.Syndrome;
for (int part = 0; part < stride; part++)
{
@@ -374,7 +400,7 @@ namespace CUETools.AccurateRip
for (int i = 0; i < npar; i++)
{
syn[i] = ar.syndrome[part, i];
syn[i] = arSyndrome[part, i];
// offset < 0
if (part < -offset * 2)
@@ -442,7 +468,7 @@ namespace CUETools.AccurateRip
{
get
{
return ar.syndrome;
return ar.Syndrome;
}
}

3
CUETools.CTDB.EACPlugin/Plugin.cs
View File

@@ -258,9 +258,8 @@ namespace AudioDataPlugIn
ctdb.Submit(
#if USEAR
(int)ar.WorstConfidence() + 1,
(int)ar.WorstTotal() + 1,
#else
1, 1,
1,
#endif
m_data.AlbumArtist,
m_data.AlbumTitle);

230
CUETools.CTDB/CUEToolsDB.cs
View File

@@ -58,7 +58,7 @@ namespace CUETools.CTDB
if (accResult == HttpStatusCode.OK)
{
total = 0;
this.total = 0;
using (Stream responseStream = resp.GetResponseStream())
{
using (XmlTextReader reader = new XmlTextReader(responseStream))
@@ -73,6 +73,7 @@ namespace CUETools.CTDB
string npar = reader["npar"];
string stride = reader["stride"];
string id = reader["id"];
string hasPatity = reader["hasparity"];
byte[] parity = null;
CDImageLayout entry_toc = null;
@@ -101,8 +102,8 @@ namespace CUETools.CTDB
}
}
entry.Close();
total += int.Parse(confidence);
entries.Add(new DBEntry(parity, 0, parity.Length, int.Parse(confidence), int.Parse(npar), uint.Parse(crc32, NumberStyles.HexNumber), id, entry_toc));
this.total += int.Parse(confidence);
entries.Add(new DBEntry(parity, 0, parity.Length, int.Parse(confidence), int.Parse(npar), int.Parse(stride), uint.Parse(crc32, NumberStyles.HexNumber), id, entry_toc, hasPatity == "1"));
} while (reader.ReadToNextSibling("entry"));
reader.Close();
}
@@ -120,21 +121,19 @@ namespace CUETools.CTDB
}
}
public void FetchDB(string url, out HttpStatusCode accResult, out byte[] contents, out int total, List<DBEntry> entries)
public void FetchDB(string url, DBEntry entry)
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "GET";
req.Proxy = proxy;
req.UserAgent = userAgent;
contents = null;
total = 0;
try
{
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
accResult = resp.StatusCode;
entry.httpStatus = resp.StatusCode;
if (accResult == HttpStatusCode.OK)
if (entry.httpStatus == HttpStatusCode.OK)
{
using (Stream responseStream = resp.GetResponseStream())
{
@@ -150,20 +149,19 @@ namespace CUETools.CTDB
memoryStream.Write(buffer, 0, count);
pos += count;
} while (count != 0);
contents = memoryStream.ToArray();
var contents = memoryStream.ToArray();
if (!Parse(contents, entry))
entry.httpStatus = HttpStatusCode.NoContent;
}
}
Parse(contents, entries, out total);
if (entries.Count == 0)
accResult = HttpStatusCode.NoContent;
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
accResult = ((HttpWebResponse)ex.Response).StatusCode;
entry.httpStatus = ((HttpWebResponse)ex.Response).StatusCode;
else
accResult = HttpStatusCode.BadRequest;
entry.httpStatus = HttpStatusCode.BadRequest;
}
}
@@ -188,87 +186,54 @@ namespace CUETools.CTDB
public string Confirm(DBEntry entry)
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlbase +
"/confirm.php?tocid=" + toc.TOCID +
"&id=" + entry.id +
"&userid=" + GetUUID() +
"&offscrc=" + verify.OffsetSafeCRC.Replace('+', '.').Replace('/', '_').Replace('=', '-').Replace("\r", "").Replace("\n", ""));
req.Method = "GET";
req.Proxy = proxy;
req.UserAgent = userAgent;
try
{
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
if (resp.StatusCode != HttpStatusCode.OK)
subResult = resp.StatusCode.ToString();
else
{
using (Stream s = resp.GetResponseStream())
using (StreamReader sr = new StreamReader(s))
subResult = sr.ReadToEnd();
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
subResult = ((HttpWebResponse)ex.Response).StatusCode.ToString();
else
subResult = "unknown error";
}
return subResult;
return Submit(1, null, null, !entry.hasParity /*&& entry.conf > 100*/, true, entry.id);
}
public string Submit(int confidence, int total, string artist, string title)
public string Submit(int confidence, string artist, string title)
{
UploadFile[] files = new UploadFile[1];
MemoryStream newcontents = new MemoryStream();
using (DBHDR FTYP = new DBHDR(newcontents, "ftyp"))
FTYP.Write("CTDB");
using (DBHDR CTDB = new DBHDR(newcontents, "CTDB"))
return Submit(confidence, artist, title, confidence > 1, false, null);
}
public string Submit(int confidence, string artist, string title, bool upload, bool confirm, string confirmid)
{
UploadFile[] files;
if (upload)
{
using (DBHDR HEAD = CTDB.HDR("HEAD"))
MemoryStream newcontents = new MemoryStream();
using (DBHDR FTYP = new DBHDR(newcontents, "ftyp"))
FTYP.Write("CTDB");
using (DBHDR CTDB = new DBHDR(newcontents, "CTDB"))
{
using (DBHDR TOTL = HEAD.HDR("TOTL")) TOTL.Write(total);
using (DBHDR VERS = HEAD.HDR("VERS")) VERS.Write(0x100);
using (DBHDR DATE = HEAD.HDR("DATE")) DATE.Write(DateTime.Now);
}
using (DBHDR DISC = CTDB.HDR("DISC"))
{
using (DBHDR TOC = DISC.HDR("TOC "))
using (DBHDR HEAD = CTDB.HDR("HEAD"))
{
using (DBHDR INFO = TOC.HDR("INFO"))
{
INFO.Write(toc.TrackCount);
INFO.Write(toc.Pregap);
}
for (int i = 1; i <= toc.TrackCount; i++)
using (DBHDR TRAK = TOC.HDR("TRAK"))
{
TRAK.Write(toc[i].IsAudio ? 1 : 0);
TRAK.Write(toc[i].Length);
}
using (DBHDR VERS = HEAD.HDR("VERS")) VERS.Write(0x101);
}
using (DBHDR DISC = CTDB.HDR("DISC"))
{
using (DBHDR CONF = DISC.HDR("CONF")) CONF.Write(confidence);
using (DBHDR NPAR = DISC.HDR("NPAR")) NPAR.Write(verify.NPAR);
using (DBHDR CRC_ = DISC.HDR("CRC ")) CRC_.Write(verify.CRC);
using (DBHDR PAR_ = DISC.HDR("PAR ")) PAR_.Write(verify.Parity);
}
if (artist != null && artist != "") using (DBHDR TAG = DISC.HDR("ART ")) TAG.Write(artist);
if (title != null && title != "") using (DBHDR TAG = DISC.HDR("nam ")) TAG.Write(title);
using (DBHDR USER = DISC.HDR("USER")) USER.Write(GetUUID());
using (DBHDR TOOL = DISC.HDR("TOOL")) TOOL.Write(userAgent);
using (DBHDR TOOL = DISC.HDR("MBID")) TOOL.Write(toc.MusicBrainzId);
using (DBHDR DATE = DISC.HDR("DATE")) DATE.Write(DateTime.Now);
using (DBHDR CONF = DISC.HDR("CONF")) CONF.Write(confidence);
using (DBHDR NPAR = DISC.HDR("NPAR")) NPAR.Write(verify.NPAR);
using (DBHDR CRC_ = DISC.HDR("CRC ")) CRC_.Write(verify.CRC);
using (var OFFS = DISC.HDR("OFFS")) OFFS.Write(verify.OffsetSafeCRC);
using (DBHDR PAR_ = DISC.HDR("PAR ")) PAR_.Write(verify.Parity);
}
newcontents.Position = 0;
files = new UploadFile[1] { new UploadFile(newcontents, "parityfile", "data.bin", "image/binary") };
}
newcontents.Position = 0;
files[0] = new UploadFile(newcontents, "uploadedfile", "data.bin", "image/binary");
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlbase + "/submit.php");
else
{
files = new UploadFile[0];
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlbase + "/submit2.php");
req.Proxy = proxy;
req.UserAgent = userAgent;
NameValueCollection form = new NameValueCollection();
if (upload)
form.Add("parityfile", "1");
if (confirm)
form.Add("confirmid", confirmid);
form.Add("tocid", toc.TOCID);
form.Add("crc32", string.Format("%08X", verify.CRC));
form.Add("crc32", ((int)verify.CRC).ToString());
form.Add("trackcrcs", verify.TrackCRCs);
form.Add("parity", Convert.ToBase64String(verify.Parity, 0, 16));
form.Add("confidence", confidence.ToString());
form.Add("trackcount", toc.TrackCount.ToString());
@@ -276,38 +241,69 @@ namespace CUETools.CTDB
form.Add("audiotracks", toc.AudioTracks.ToString());
form.Add("trackoffsets", toc.TrackOffsets);
form.Add("userid", GetUUID());
form.Add("agent", userAgent);
if (artist != null && artist != "") form.Add("artist", artist);
if (title != null && title != "") form.Add("title", title);
HttpWebResponse resp = uploadHelper.Upload(req, files, form);
using (Stream s = resp.GetResponseStream())
using (StreamReader sr = new StreamReader(s))
subResult = sr.ReadToEnd();
var ExceptionStatus = WebExceptionStatus.Pending;
string ExceptionMessage = null;
HttpStatusCode ResponseStatus = HttpStatusCode.OK;
try
{
using (HttpWebResponse resp = uploadHelper.Upload(req, files, form))
{
ExceptionStatus = WebExceptionStatus.ProtocolError;
ResponseStatus = resp.StatusCode;
if (ResponseStatus == HttpStatusCode.OK)
{
ExceptionStatus = WebExceptionStatus.Success;
using (Stream s = resp.GetResponseStream())
using (StreamReader sr = new StreamReader(s))
subResult = sr.ReadToEnd();
return subResult;
}
}
}
catch (WebException ex)
{
ExceptionStatus = ex.Status;
ExceptionMessage = ex.Message;
if (ExceptionStatus == WebExceptionStatus.ProtocolError)
ResponseStatus = (ex.Response as HttpWebResponse).StatusCode;
}
subResult = ExceptionStatus == WebExceptionStatus.Success ? null :
ExceptionStatus != WebExceptionStatus.ProtocolError ? ("database access error: " + (ExceptionMessage ?? ExceptionStatus.ToString())) :
ResponseStatus != HttpStatusCode.NotFound ? "database access error: " + ResponseStatus.ToString() :
"disk not present in database";
return subResult;
}
private void Parse(byte[] contents, List<DBEntry> entries, out int total)
private bool Parse(byte[] contents, DBEntry entry)
{
if (contents.Length == entry.npar * entry.stride)
{
entry.parity = contents;
entry.pos = 0;
entry.len = contents.Length;
return true;
}
ReadDB rdr = new ReadDB(contents);
total = 0;
int end;
string hdr = rdr.ReadHDR(out end);
uint magic = rdr.ReadUInt();
if (hdr != "ftyp" || magic != 0x43544442 || end != rdr.pos)
throw new Exception("invalid CTDB file");
hdr = rdr.ReadHDR(out end);
if (hdr != "CTDB" || end != contents.Length)
if (hdr != "CTDB" || end != contents.Length)
throw new Exception("invalid CTDB file");
hdr = rdr.ReadHDR(out end);
if (hdr != "HEAD")
if (hdr != "HEAD")
throw new Exception("invalid CTDB file");
int endHead = end;
while (rdr.pos < endHead)
{
hdr = rdr.ReadHDR(out end);
if (hdr == "TOTL")
total = rdr.ReadInt();
rdr.pos = end;
}
rdr.pos = endHead;
@@ -320,8 +316,7 @@ namespace CUETools.CTDB
continue;
}
int endDisc = end;
uint crc = 0;
int parPos = 0, parLen = 0, conf = 0, npar = 0;
int parPos = 0, parLen = 0;
while (rdr.pos < endDisc)
{
hdr = rdr.ReadHDR(out end);
@@ -330,27 +325,26 @@ namespace CUETools.CTDB
parPos = rdr.pos;
parLen = end - rdr.pos;
}
else if (hdr == "CRC ")
crc = rdr.ReadUInt();
else if (hdr == "CONF")
conf = rdr.ReadInt();
else if (hdr == "NPAR")
npar = rdr.ReadInt();
rdr.pos = end;
}
if (parPos != 0 && npar >= 2 && npar <= 16 && conf >= 0)
//if (parPos != 0 && npar >= 2 && npar <= 16 && conf != 0)
entries.Add(new DBEntry(contents, parPos, parLen, conf, npar, crc, null, null));
if (parPos != 0)
{
entry.parity = contents;
entry.pos = parPos;
entry.len = parLen;
return true;
}
}
return false;
}
public void DoVerify()
{
foreach (DBEntry entry in entries)
{
if (entry.toc.Pregap != toc.Pregap || entry.toc.AudioLength != toc.AudioLength)
if (entry.toc.Pregap != toc.Pregap || entry.toc.AudioLength != toc.AudioLength || entry.stride != verify.Stride / 2)
{
entry.hasErrors = false;
entry.hasErrors = true;
entry.canRecover = false;
continue;
}
@@ -358,16 +352,18 @@ namespace CUETools.CTDB
entry.canRecover = false;
else if (entry.hasErrors)
{
byte[] contents2;
int total2;
List<DBEntry> entries2 = new List<DBEntry>();
FetchDB(string.Format("{0}/repair.php?tocid={1}&id={2}", urlbase, toc.TOCID, entry.id), out entry.httpStatus, out contents2, out total2, entries2);
if (entry.httpStatus != HttpStatusCode.OK)
if (!entry.hasParity)
entry.canRecover = false;
else
{
entry.repair = verify.VerifyParity(entries2[0].npar, contents2, entries2[0].pos, entries2[0].len, entry.offset);
entry.canRecover = entry.repair.CanRecover;
FetchDB(string.Format("{0}/repair.php?tocid={1}&id={2}", urlbase, toc.TOCID, entry.id), entry);
if (entry.httpStatus != HttpStatusCode.OK)
entry.canRecover = false;
else
{
entry.repair = verify.VerifyParity(entry.npar, entry.parity, entry.pos, entry.len, entry.offset);
entry.canRecover = entry.repair.CanRecover;
}
}
}
}
@@ -378,7 +374,7 @@ namespace CUETools.CTDB
int npar = 8;
foreach (DBEntry entry in entries)
npar = Math.Max(npar, entry.npar);
verify = new CDRepairEncode(ar, 10 * 588 * 2, npar, entries.Count > 0, encode);
verify = new CDRepairEncode(ar, 10 * 588 * 2, npar);
}
public CDImageLayout TOC
@@ -504,6 +500,7 @@ namespace CUETools.CTDB
public int len;
public int conf;
public int npar;
public int stride;
public int offset;
public uint crc;
public bool hasErrors;
@@ -512,8 +509,9 @@ namespace CUETools.CTDB
public HttpStatusCode httpStatus;
public string id;
public CDImageLayout toc;
public bool hasParity;
public DBEntry(byte[] parity, int pos, int len, int conf, int npar, uint crc, string id, CDImageLayout toc)
public DBEntry(byte[] parity, int pos, int len, int conf, int npar, int stride, uint crc, string id, CDImageLayout toc, bool hasParity)
{
this.parity = parity;
this.id = id;
@@ -522,7 +520,9 @@ namespace CUETools.CTDB
this.conf = conf;
this.crc = crc;
this.npar = npar;
this.stride = stride;
this.toc = toc;
this.hasParity = hasParity;
}
public string Status

BIN
CUETools.Parity/Galois.cs
View File

Binary file not shown.

2
CUETools.Processor/Processor.cs
View File

@@ -6316,7 +6316,7 @@ string status = processor.Go();
CTDB.SubStatus = "will not submit";
return GenerateAccurateRipStatus();
}
CTDB.Submit((int)ArVerify.WorstConfidence(), (int)ArVerify.WorstTotal(), Artist, Title);
CTDB.Submit((int)ArVerify.WorstConfidence(), Artist, Title);
return GenerateAccurateRipStatus();
}
case "repair":

6
CUETools.TestHelpers/TestImageGenerator.cs
View File

@@ -70,13 +70,13 @@ namespace CUETools.TestHelpers
return ar;
}
public CDRepairEncode CreateCDRepairEncode(int stride, int npar, bool do_verify, bool do_encode)
public CDRepairEncode CreateCDRepairEncode(int stride, int npar)
{
var ar = new AccurateRipVerify(toc, null);
var encode = new CDRepairEncode(ar, stride, npar, do_verify, do_encode);
var encode = new CDRepairEncode(ar, stride, npar);
ar.Position = start;
Write(ar);
ar.Close();
//ar.Close();
return encode;
}

165
CUETools/CUETools.TestParity/CDRepairDecodeTest.cs
View File

@@ -4,6 +4,7 @@ using CUETools.Codecs;
using CUETools.CDImage;
using CUETools.AccurateRip;
using CUETools.TestHelpers;
using CUETools.Parity;
namespace CUETools.TestParity
{
@@ -62,7 +63,7 @@ namespace CUETools.TestParity
public static void MyClassInitialize(TestContext testContext)
{
generator = new TestImageGenerator("0 9801", seed, 32 * 588, 0);
encode = generator.CreateCDRepairEncode(stride, npar, false, true);
encode = generator.CreateCDRepairEncode(stride, npar);
}
//
//Use ClassCleanup to run code after all tests in a class have run
@@ -102,7 +103,7 @@ namespace CUETools.TestParity
[TestMethod()]
public void CDRepairDecodeOriginalTest()
{
var decode = generator.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -117,7 +118,7 @@ namespace CUETools.TestParity
public void CDRepairDecodeOriginalWithPregapTest()
{
var generator2 = new TestImageGenerator("32 9833", seed, 0, 0);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -135,7 +136,7 @@ namespace CUETools.TestParity
public void CDRepairDecodeModifiedTest()
{
var generator2 = new TestImageGenerator("0 9801", seed, 32 * 588, errors);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -155,7 +156,7 @@ namespace CUETools.TestParity
public void CDRepairDecodePositiveOffsetTest()
{
var generator2 = new TestImageGenerator("0 9801", seed, 32 * 588 + offset, 0);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -170,7 +171,7 @@ namespace CUETools.TestParity
public void CDRepairDecodeNegativeOffsetTest()
{
var generator2 = new TestImageGenerator("0 9801", seed, 32 * 588 - offset, 0);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -185,7 +186,7 @@ namespace CUETools.TestParity
public void CDRepairDecodePositiveOffsetErrorsTest()
{
var generator2 = new TestImageGenerator("0 9801", seed, 32 * 588 + offset, errors);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors));
@@ -205,7 +206,7 @@ namespace CUETools.TestParity
public void CDRepairDecodeNegativeOffsetErrorsTest()
{
var generator2 = new TestImageGenerator("0 999 9801", seed, 32 * 588 - offset, errors);
var decode = generator2.CreateCDRepairEncode(stride, npar, true, false);
var decode = generator2.CreateCDRepairEncode(stride, npar);
int actualOffset;
bool hasErrors;
Assert.IsTrue(decode.FindOffset(encode.NPAR, encode.Parity, 0, encode.CRC, out actualOffset, out hasErrors), "couldn't find offset");
@@ -218,29 +219,131 @@ namespace CUETools.TestParity
Assert.AreEqual<uint>(encode.CRC, fix.CRC);
}
//[TestMethod]
//public void CDRepairDecodeSpeedTest()
//{
// var generator = new TestImageGenerator("0 75000", seed, 0, 0);
// var encode = generator.CreateCDRepairEncode(stride, npar, true, false);
// var syn = new byte[64];
// for (int i = 0; i < 4; i++)
// for (int j = 0; j < 8; j++)
// {
// syn[(i * 8 + j) * 2] = (byte)encode.Syndrome[i, j];
// syn[(i * 8 + j) * 2 + 1] = (byte)(encode.Syndrome[i, j] >> 8);
// }
// Assert.AreEqual<string>("YJPyo4+KY35P+DpljMplMGbMWXmpvhkdDOCKeEo4NDoRPPW7D0cv8hmLb7yZujp0sVg/6AEWKY5QrDKkiYp0Zw==",
// Convert.ToBase64String(syn));
//}
[TestMethod]
public void GFMiscTest()
{
var g16 = Galois16.instance;
CollectionAssert.AreEqual(new int[] { 5, 33657, 33184, 33657, 5 }, g16.gfconv(new int[] { 1, 2, 3 }, new int[] { 4, 3, 2 }));
CollectionAssert.AreEqual(new int[] { 5, 6, 1774, 4, 5 }, g16.gfconv(new int[] { 1, 2, 3 }, new int[] { 4, -1, 2 }));
//[TestMethod]
//public void CDRepairEncodeSpeedTest()
//{
// var generator = new TestImageGenerator("0 75000", seed, 0, 0);
// var encode = generator.CreateCDRepairEncode(stride, npar, false, true);
// Assert.AreEqual<string>("CWgEDNLjSi22nIOyaeyp+12R3UCVWlzIb+nbv8XWXg9YEhkHxYr8xqrr1+hIbFwKNEXnj0esJrKbiW3XGbHsYw==",
// Convert.ToBase64String(encode.Parity, 0, 64));
//}
var g8 = Galois81D.instance;
Assert.AreEqual(111, g8.gfadd(11, 15));
Assert.AreEqual(11, g8.gfadd(11, -1));
Assert.AreEqual(25, g8.gfadd(1, 0));
var S = new int[8] { -1, -1, -1, -1, -1, -1, -1, -1 };
var received = new int[] { 2, 4, 1, 3, 5 };
for (int ii = 0; ii < 8; ii++)
for (int x = 0; x < 5; x++)
S[ii] = g8.gfadd(S[ii], g8.gfmul(received[x], g8.gfpow(ii + 1, x)));
CollectionAssert.AreEqual(S, new int[] { 219, 96, 208, 202, 116, 211, 182, 129 });
//S[ii] ^= received[x] * a ^ ((ii + 1) * x);
//S[0] ^= received[0] * a ^ (1 * 0);
//S[0] ^= received[1] * a ^ (1 * 1);
//S[0] ^= received[2] * a ^ (1 * 2);
//S[1] ^= received[0] * a ^ (2 * 0);
//S[1] ^= received[1] * a ^ (2 * 1);
//S[1] ^= received[2] * a ^ (2 * 2);
received = g8.toExp(received);
for (int ii = 0; ii < 8; ii++)
{
S[ii] = 0;
for (int x = 0; x < 5; x++)
S[ii] ^= g8.mulExp(received[x], ((ii + 1) * x) % 255);
}
S = g8.toLog(S);
CollectionAssert.AreEqual(S, new int[] { 219, 96, 208, 202, 116, 211, 182, 129 });
}
/// <summary>
///A test for CRC parralelism
///</summary>
[TestMethod()]
public void CDRepairSplitTest()
{
var seed = 723722;
var ar0 = new TestImageGenerator("13 68 99 136", seed, 0, 0).CreateCDRepairEncode(stride, npar);
var splits = new int[] { 1, 13 * 588 - 1, 13 * 588, 13 * 588 + 1, 30 * 588 - 1, 30 * 588, 30 * 588 + 1, 68 * 588 - 1, 68 * 588, 68 * 588 + 1 };
foreach (int split in splits)
{
var ar1 = new TestImageGenerator("13 68 99 136", seed, 0, 0, 0, split).CreateCDRepairEncode(stride, npar);
var ar2 = new TestImageGenerator("13 68 99 136", seed, 0, 0, split, (int)ar0.FinalSampleCount).CreateCDRepairEncode(stride, npar);
ar1.AR.Combine(ar2.AR, split, (int)ar0.FinalSampleCount);
var offsets = new int[] { 0, -1, 1, -2, 2, -3, 3, -4, 4, -11, 11, -256, 256, -588, 588, 1 - 588 * 5, 588 * 5 - 1 };
string message = "split = " + CDImageLayout.TimeToString((uint)split / 588) + "." + (split % 588).ToString();
Assert.AreEqual(ar0.CRC, ar1.CRC, "CRC was not set correctly, " + message);
CollectionAssert.AreEqual(ar0.Parity, ar1.Parity, "Parity was not set correctly, " + message);
}
}
/// <summary>
///A test for CRC parralelism speed
///</summary>
[TestMethod()]
public unsafe void CDRepairSplitSpeedTest()
{
var seed = 723722;
var split = 20 * 588;
var ar1 = new TestImageGenerator("13 68 99 136", seed, 0, 0, 0, split).CreateCDRepairEncode(stride, npar);
var ar2 = new TestImageGenerator("13 68 99 136", seed, 0, 0, split, (int)ar1.FinalSampleCount).CreateCDRepairEncode(stride, npar);
for (int i = 0; i < 20; i++)
ar1.AR.Combine(ar2.AR, split, (int)ar1.FinalSampleCount);
}
/// <summary>
///A test for Syndrome2Parity speed
///</summary>
[TestMethod()]
public unsafe void CDRepairSyndrome2ParitySpeedTest()
{
byte[] parityCopy = new byte[encode.Parity.Length];
for (int t = 0; t < 100; t++)
{
fixed (byte* p = encode.Parity, p1 = parityCopy)
fixed (ushort* syn = encode.Syndrome)
{
ushort* p2 = (ushort*)p1;
for (int i = 0; i < stride; i++)
Galois16.instance.Syndrome2Parity(syn + i * npar, p2 + i * npar, npar);
}
}
CollectionAssert.AreEqual(encode.Parity, parityCopy);
}
[TestMethod]
public unsafe void CDRepairEncodeSynParTest()
{
byte[] parityCopy = new byte[encode.Parity.Length];
fixed(byte * p = encode.Parity, p1 = parityCopy)
fixed (ushort *syn = encode.Syndrome)
{
ushort* p2 = (ushort*)p1;
for (int i = 0; i < stride; i++)
Galois16.instance.Syndrome2Parity(syn + i * npar, p2 + i * npar, npar);
}
CollectionAssert.AreEqual(encode.Parity, parityCopy);
}
[TestMethod]
public void CDRepairEncodeSpeedTest()
{
var generator = new TestImageGenerator("0 75000", seed, 0, 0);
var encode = generator.CreateCDRepairEncode(stride, npar);
Assert.AreEqual<string>("CWgEDNLjSi22nIOyaeyp+12R3UCVWlzIb+nbv8XWXg9YEhkHxYr8xqrr1+hIbFwKNEXnj0esJrKbiW3XGbHsYw==",
Convert.ToBase64String(encode.Parity, 0, 64));
var syndrome = encode.Syndrome;
var bsyn = new byte[64];
for (int i = 0; i < 4; i++)
for (int j = 0; j < 8; j++)
{
bsyn[(i * 8 + j) * 2] = (byte)syndrome[i, j];
bsyn[(i * 8 + j) * 2 + 1] = (byte)(syndrome[i, j] >> 8);
}
Assert.AreEqual<string>("YJPyo4+KY35P+DpljMplMGbMWXmpvhkdDOCKeEo4NDoRPPW7D0cv8hmLb7yZujp0sVg/6AEWKY5QrDKkiYp0Zw==",
Convert.ToBase64String(bsyn));
}
}
}

1
CUETools/CUETools.TestParity/CUETools.TestParity.csproj
View File

@@ -31,6 +31,7 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />