claunia/cuetools.net
1
0
Fork 0
mirror of https://github.com/claunia/cuetools.net.git synced 2025-12-16 18:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

CD parity information database

Browse Source
This commit is contained in:
chudov
2010-02-23 23:24:54 +00:00
parent 21b3ef3d15
commit cc6d2ff950
4 changed files with 466 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CUETools.CDRepair/CDRepair.cs
View File

@@ -437,6 +437,7 @@ namespace CUETools.CDRepair
ushort* par2 = (ushort*)par2ptr;
int* _sigma = stackalloc int[npar];
int* _errpos = stackalloc int[npar];
int* syn = stackalloc int[npar];
bool foundOffset = false;
for (int allowed_errors = 0; allowed_errors < npar / 2 && !foundOffset; allowed_errors++)
@@ -452,7 +453,6 @@ namespace CUETools.CDRepair
{
int err = 0;
int part = (part2 + stride - offset * 2) % stride;
int* syn = stackalloc int[npar];
for (int i = 0; i < npar; i++)
{

192
CUETools.CTDB/CUEToolsDB.cs
View File

@@ -14,13 +14,16 @@ namespace CUETools.CTDB
{
public class CUEToolsDB
{
const string urlbase = "http://db.cuetools.net";
private CDRepairEncode verify;
private CDImageLayout toc;
private HttpStatusCode accResult;
private string id;
private string urlfolder;
private string fullid;
private string subResult;
private byte[] contents;
private int pos;
private int length;
private int total;
List<DBEntry> entries = new List<DBEntry>();
@@ -52,12 +55,19 @@ namespace CUETools.CTDB
discId2 = UInt32.Parse(n[1], NumberStyles.HexNumber);
cddbDiscId = UInt32.Parse(n[2], NumberStyles.HexNumber);
string url = String.Format("http://db.cuetools.net/parity/{0:x}/{1:x}/{2:x}/dBCT-{3:d3}-{4:x8}-{5:x8}-{6:x8}.bin",
discId1 & 0xF, discId1 >> 4 & 0xF, discId1 >> 8 & 0xF, toc.AudioTracks, discId1, discId2, cddbDiscId);
fullid = string.Format("{0:d3}-{1:x8}-{2:x8}-{3:x8}", toc.AudioTracks, discId1, discId2, cddbDiscId);
urlfolder = string.Format("{0}/parity/{1:x}/{2:x}/{3:x}/{4}", urlbase, discId1 & 0xF, discId1 >> 4 & 0xF, discId1 >> 8 & 0xF, fullid);
FetchDB(string.Format("{0}/ctdb.bin", urlfolder), out accResult, out contents, out total, entries);
}
public void FetchDB(string url, out HttpStatusCode accResult, out byte[] contents, out int total, List<DBEntry> entries)
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "GET";
req.Proxy = proxy;
contents = null;
total = 0;
try
{
@@ -71,20 +81,23 @@ namespace CUETools.CTDB
using(MemoryStream memoryStream = new MemoryStream())
{
byte[] buffer = new byte[16536];
int count = 0;
int count = 0, pos = 0;
do
{
if (uploadHelper.onProgress != null)
uploadHelper.onProgress(url, new UploadProgressEventArgs(req.RequestUri.AbsoluteUri, ((double)pos) / resp.ContentLength));
count = responseStream.Read(buffer, 0, buffer.Length);
memoryStream.Write(buffer, 0, count);
pos += count;
} while (count != 0);
contents = memoryStream.ToArray();
}
}
}
Parse();
Parse(contents, entries, out total);
if (entries.Count == 0)
accResult = HttpStatusCode.NoContent;
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
@@ -113,19 +126,8 @@ namespace CUETools.CTDB
public string Submit(int confidence, int total)
{
if (id == null)
if (fullid == null)
throw new Exception("no id");
// Calculate the three disc ids used by AR
uint discId1 = 0;
uint discId2 = 0;
uint cddbDiscId = 0;
string[] n = id.Split('-');
if (n.Length != 3)
throw new Exception("Invalid accurateRipId.");
discId1 = UInt32.Parse(n[0], NumberStyles.HexNumber);
discId2 = UInt32.Parse(n[1], NumberStyles.HexNumber);
cddbDiscId = UInt32.Parse(n[2], NumberStyles.HexNumber);
UploadFile[] files = new UploadFile[1];
MemoryStream newcontents = new MemoryStream();
@@ -135,10 +137,9 @@ namespace CUETools.CTDB
{
using (DBHDR HEAD = CTDB.HDR("HEAD"))
{
HEAD.Write(0x100); // version
HEAD.Write(1); // disc count
HEAD.Write(total); // total submissions
HEAD.Write(DateTime.Now); // date
using (DBHDR TOTL = HEAD.HDR("TOTL")) TOTL.Write(total);
using (DBHDR VERS = HEAD.HDR("VERS")) VERS.Write(0x100);
using (DBHDR DATE = HEAD.HDR("DATE")) DATE.Write(DateTime.Now);
}
using (DBHDR DISC = CTDB.HDR("DISC"))
{
@@ -167,11 +168,11 @@ namespace CUETools.CTDB
}
newcontents.Position = 0;
files[0] = new UploadFile(newcontents, "uploadedfile", "data.bin", "image/binary");
HttpWebRequest req = (HttpWebRequest)WebRequest.Create("http://db.cuetools.net/uploader.php");
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlbase + "/uploader2.php");
req.Proxy = proxy;
req.UserAgent = "CUETools 205";
NameValueCollection form = new NameValueCollection();
form.Add("id", String.Format("{0:d3}-{1:x8}-{2:x8}-{3:x8}", toc.AudioTracks, discId1, discId2, cddbDiscId));
form.Add("id", fullid);
HttpWebResponse resp = uploadHelper.Upload(req, files, form);
using (Stream s = resp.GetResponseStream())
using (StreamReader sr = new StreamReader(s))
@@ -179,87 +180,57 @@ namespace CUETools.CTDB
return subResult;
}
private string ReadHDR(out int end)
private void Parse(byte[] contents, List<DBEntry> entries, out int total)
{
int size = ReadInt();
string res = Encoding.ASCII.GetString(contents, pos, 4);
pos += 4;
end = pos + size - 8;
return res;
}
ReadDB rdr = new ReadDB(contents);
private int ReadInt()
{
int value =
(contents[pos + 3] +
(contents[pos + 2] << 8) +
(contents[pos + 1] << 16) +
(contents[pos + 0] << 24));
pos += 4;
return value;
}
private uint ReadUInt()
{
uint value =
((uint)contents[pos + 3] +
((uint)contents[pos + 2] << 8) +
((uint)contents[pos + 1] << 16) +
((uint)contents[pos + 0] << 24));
pos += 4;
return value;
}
private void Parse()
{
if (accResult != HttpStatusCode.OK)
return;
pos = 0;
total = 0;
int end;
string hdr = ReadHDR(out end);
uint magic = ReadUInt();
if (hdr != "ftyp" || magic != 0x43544442 || end != pos)
string hdr = rdr.ReadHDR(out end);
uint magic = rdr.ReadUInt();
if (hdr != "ftyp" || magic != 0x43544442 || end != rdr.pos)
throw new Exception("invalid CTDB file");
hdr = ReadHDR(out end);
hdr = rdr.ReadHDR(out end);
if (hdr != "CTDB" || end != contents.Length)
throw new Exception("invalid CTDB file");
hdr = ReadHDR(out end);
hdr = rdr.ReadHDR(out end);
if (hdr != "HEAD")
throw new Exception("invalid CTDB file");
uint version = ReadUInt();
int discCount = ReadInt();
total = ReadInt();
if (discCount <= 0 || version >= 0x200)
throw new Exception("invalid CTDB file");
// date
pos = end;
while (pos < contents.Length)
int endHead = end;
while (rdr.pos < endHead)
{
hdr = ReadHDR(out end);
hdr = rdr.ReadHDR(out end);
if (hdr == "TOTL")
total = rdr.ReadInt();
rdr.pos = end;
}
rdr.pos = end;
while (rdr.pos < contents.Length)
{
hdr = rdr.ReadHDR(out end);
if (hdr != "DISC")
{
pos = end;
rdr.pos = end;
continue;
}
int endDisc = end;
uint crc = 0;
int parPos = 0, parLen = 0, conf = 0, npar = 0;
while (pos < endDisc)
while (rdr.pos < endDisc)
{
hdr = ReadHDR(out end);
hdr = rdr.ReadHDR(out end);
if (hdr == "PAR ")
{
parPos = pos;
parLen = end - pos;
parPos = rdr.pos;
parLen = end - rdr.pos;
}
else if (hdr == "CRC ")
crc = ReadUInt();
crc = rdr.ReadUInt();
else if (hdr == "CONF")
conf = ReadInt();
conf = rdr.ReadInt();
else if (hdr == "NPAR")
npar = ReadInt();
pos = end;
npar = rdr.ReadInt();
rdr.pos = end;
}
if (parPos != 0 && npar >= 2 && npar <= 16 && conf >= 0)
//if (parPos != 0 && npar >= 2 && npar <= 16 && conf != 0)
@@ -275,11 +246,20 @@ namespace CUETools.CTDB
entry.canRecover = false;
else if (entry.hasErrors)
{
entry.repair = verify.VerifyParity(entry.npar, contents, entry.pos, entry.len, entry.offset);
byte[] contents2;
int total2;
List<DBEntry> entries2 = new List<DBEntry>();
FetchDB(string.Format("{0}/{1:x8}.bin", urlfolder, entry.crc), out entry.httpStatus, out contents2, out total2, entries2);
if (entry.httpStatus != HttpStatusCode.OK)
entry.canRecover = false;
else
{
entry.repair = verify.VerifyParity(entries2[0].npar, contents2, entries2[0].pos, entries2[0].len, entry.offset);
entry.canRecover = entry.repair.CanRecover;
}
}
}
}
public void Init(bool encode)
{
@@ -392,6 +372,7 @@ namespace CUETools.CTDB
public bool hasErrors;
public bool canRecover;
public CDRepairFix repair;
public HttpStatusCode httpStatus;
public DBEntry(int pos, int len, int conf, int npar, uint crc)
{
@@ -410,11 +391,56 @@ namespace CUETools.CTDB
return string.Format("verified OK, confidence {0}", conf);
if (canRecover)
return string.Format("contains {1} correctable errors, confidence {0}", conf, repair.CorrectableErrors);
if (httpStatus == HttpStatusCode.OK)
return "could not be verified";
return "could not be verified: " + httpStatus.ToString();
}
}
}
internal class ReadDB
{
byte[] contents;
public int pos;
public ReadDB(byte[] contents)
{
this.contents = contents;
pos = 0;
}
public string ReadHDR(out int end)
{
int size = ReadInt();
string res = Encoding.ASCII.GetString(contents, pos, 4);
pos += 4;
end = pos + size - 8;
return res;
}
public int ReadInt()
{
int value =
(contents[pos + 3] +
(contents[pos + 2] << 8) +
(contents[pos + 1] << 16) +
(contents[pos + 0] << 24));
pos += 4;
return value;
}
public uint ReadUInt()
{
uint value =
((uint)contents[pos + 3] +
((uint)contents[pos + 2] << 8) +
((uint)contents[pos + 1] << 16) +
((uint)contents[pos + 0] << 24));
pos += 4;
return value;
}
}
internal class DBHDR : IDisposable
{
private long lenOffs;

359
CUETools.CTDB/uploader.php
View File

@@ -1,14 +1,355 @@
<?php
$target_path = "parity/";
$target_path = $target_path . basename( $_FILES['uploadedfile']['name']);
$source_path = $_FILES['uploadedfile']['tmp_name'];
if(move_uploaded_file($_FILES['uploadedfile']['tmp_name'], $target_path)) {
echo "The file ". basename( $_FILES['uploadedfile']['name']).
" has been uploaded";
} else{
echo $target_path, ":", $source_path;
echo "There was an error uploading the file, please try again!";
/**
* Convert php.ini shorthands to byte
*
* @author <gilthans dot NO dot SPAM at gmail dot com>
* @link http://de3.php.net/manual/en/ini.core.php#79564
*/
function php_to_byte($v){
$l = substr($v, -1);
$ret = substr($v, 0, -1);
switch(strtoupper($l)){
case 'P':
$ret *= 1024;
case 'T':
$ret *= 1024;
case 'G':
$ret *= 1024;
case 'M':
$ret *= 1024;
case 'K':
$ret *= 1024;
break;
}
return $ret;
}
// Return the human readable size of a file
// @param int $size a file size
// @param int $dec a number of decimal places
function filesize_h($size, $dec = 1)
{
$sizes = array('byte(s)', 'kb', 'mb', 'gb');
$count = count($sizes);
$i = 0;
while ($size >= 1024 && ($i < $count - 1)) {
$size /= 1024;
$i++;
}
return round($size, $dec) . ' ' . $sizes[$i];
}
function BigEndian2Int($byte_word, $signed = false) {
$int_value = 0;
$byte_wordlen = strlen($byte_word);
for ($i = 0; $i < $byte_wordlen; $i++) {
$int_value += ord($byte_word{$i}) * pow(256, ($byte_wordlen - 1 - $i));
}
if ($signed) {
$sign_mask_bit = 0x80 << (8 * ($byte_wordlen - 1));
if ($int_value & $sign_mask_bit) {
$int_value = 0 - ($int_value & ($sign_mask_bit - 1));
}
}
return $int_value;
}
function LittleEndian2String($number, $minbytes=1, $synchsafe=false) {
$intstring = '';
while ($number > 0) {
if ($synchsafe) {
$intstring = $intstring.chr($number & 127);
$number >>= 7;
} else {
$intstring = $intstring.chr($number & 255);
$number >>= 8;
}
}
return str_pad($intstring, $minbytes, "\x00", STR_PAD_RIGHT);
}
function BigEndian2String($number, $minbytes=1, $synchsafe=false) {
return strrev(LittleEndian2String($number, $minbytes, $synchsafe));
}
function unparse_atom($fp, $atom)
{
// printf('unparse_atom(%s)<br>', $atom['name']);
$offset = ftell($fp);
fwrite($fp, BigEndian2String(0, 4));
fwrite($fp, $atom['name']);
if ($atom['subatoms'])
foreach ($atom['subatoms'] as $subatom)
unparse_atom($fp, $subatom);
else if ($atom['value'])
fwrite($fp, $atom['value']);
else
die(sprintf("couldn't write long atom %s: size %d", $atom['name'], $atom['size']));
$pos = ftell($fp);
fseek($fp, $offset, SEEK_SET);
fwrite($fp, BigEndian2String($pos - $offset, 4));
fseek($fp, $pos, SEEK_SET);
}
function parse_container_atom($fp, $offset, $len)
{
// printf('parse_container_atom(%d, %d)<br>', $offset, $len);
$atoms = false;
$fin = $offset + $len;
while ($offset < $fin) {
fseek($fp, $offset, SEEK_SET);
$atom_header = fread($fp, 8);
$atom_size = BigEndian2Int(substr($atom_header, 0, 4));
$atom_name = substr($atom_header, 4, 4);
$atom['name'] = $atom_name;
$atom['size'] = $atom_size - 8;
$atom['offset'] = $offset + 8;
if ($atom_size - 8 <= 32)
$atom['value'] = fread($fp, $atom_size - 8);
else
$atom['value'] = false;
// echo $offset, ":", $atom_size, ":", $atom_name, '<br>';
if ($atom_name == 'CTDB' || $atom_name == 'DISC' || $atom_name == 'TOC ')
{
$atom['subatoms'] = parse_container_atom($fp, $offset + 8, $atom_size - 8);
foreach ($atom['subatoms'] as $param)
switch ($param['name']) {
case 'HEAD':
case 'CRC ':
case 'NPAR':
case 'CONF':
case 'PAR ':
$atom[$param['name']] = $param;
break;
case 'DISC':
$atom['discs'][] = $param;
break;
}
} else
$atom['subatoms'] = false;
switch ($atom_name)
{
case 'CRC ':
case 'NPAR':
case 'CONF':
$atom['int'] = BigEndian2Int($atom['value']);
break;
}
$offset += $atom_size;
$atoms[] = $atom;
}
if ($offset > $fin)
die("bad atom");
return $atoms;
}
function get_chunk_offset($fp, $offset, $maxlen, $names, $namepos, &$res, &$len)
{
// printf('get_chunk_offset(%d, %d, [%d]%s)<br>', $offset, $maxlen, $namepos, $names[$namepos]);
$subatoms = parse_container_atom($fp, $offset, $maxlen);
if (!$subatoms) return -1;
foreach($subatoms as $atom)
if ($atom['name'] == $names[$namepos])
{
if ($namepos + 1 >= count($names))
{
$res = $atom['offset'];
$len = $atom['size'];
return 0;
}
return get_chunk_offset($fp, $atom['offset'], $atom['size'], $names, $namepos + 1, $res, $len);
}
return -1;
}
function chunk_offset($fp, $offset, $maxlen, $path, &$res, &$len)
{
// printf('chunk_offset(%d, %d, %s)<br>', $offset, $maxlen, $path);
return get_chunk_offset($fp, $offset, $maxlen, explode(".", $path), 0, $res, $len);
}
function read_chunk($fp, $offset, $maxlen, $path, $len = 32)
{
// printf('read_chunk(%d, %d, %s)<br>', $offset, $maxlen, $path);
if (chunk_offset($fp, $offset, $maxlen, $path, $chunk_offset, $chunk_length) < 0) return;
if ($chunk_length > $len) return;
fseek($fp, $chunk_offset, SEEK_SET);
return fread($fp, $chunk_length);
}
function read_int($fp, $offset, $len, $path)
{
// printf('read_int(%d, %d, %s)<br>', $offset, $len, $path);
return BigEndian2Int(read_chunk($fp, $offset, $len, $path, 4));
}
function copy_data($srcfp, $srcoffset, $dstfp, $dstoffset, $length)
{
fseek($srcfp, $srcoffset, SEEK_SET);
fseek($dstfp, $dstoffset, SEEK_SET);
fwrite($dstfp, fread($srcfp, $length));
}
$file = $_FILES['uploadedfile'];
//echo $file['name'], ini_get('upload_max_filesize');
// give info on PHP catched upload errors
if($file['error']) switch($file['error']){
case 1:
case 2:
echo sprintf($lang['uploadsize'],
filesize_h(php_to_byte(ini_get('upload_max_filesize'))));
echo "Error ", $file['error'];
return;
default:
echo $lang['uploadfail'];
echo "Error ", $file['error'];
}
//if ($_SERVER['HTTP_USER_AGENT'] != "CUETools 205") {
// echo "user agent ", $_SERVER['HTTP_USER_AGENT'], " is not allowed";
// return;
//}
$tmpname = $file['tmp_name'];
$size = (@file_exists($tmpname)) ? filesize($tmpname) : 0;
if ($size == 0) {
echo "no file uploaded";
return;
}
$id = $_POST['id'];
$err = sscanf($id, "%03d-%04x%04x-%04x%04x-%04x%04x", $tracks, $id1a, $id1b, $id2a, $id2b, $cddbida, $cddbidb);
$parsedid = sprintf("%03d-%04x%04x-%04x%04x-%04x%04x", $tracks, $id1a, $id1b, $id2a, $id2b, $cddbida, $cddbidb);
if ($id != $parsedid) {
echo "bad id ", $id;
return;
}
$fp = fopen($tmpname, 'rb');
if (chunk_offset($fp, 0, $size, 'CTDB.DISC', $disc_offset, $disc_length) < 0)
die("bad file");
$head = read_chunk($fp, 0, $size, 'CTDB.HEAD', 20);
$crc = read_int($fp, $disc_offset, $disc_length, 'CRC ');
$npar = read_int($fp, $disc_offset, $disc_length, 'NPAR');
$tool = read_chunk($fp, $disc_offset, $disc_length, 'TOOL');
$version = BigEndian2Int(substr($head,0,4));
$disccount = BigEndian2Int(substr($head,4,4));
$total = BigEndian2Int(substr($head,8,4));
fclose($fp);
$target_path = sprintf("parity/%x/%x/%x", $id1b & 15, ($id1b >> 4) & 15, ($id1b >> 8) & 15);
$target_file = sprintf("%s/dBCT-%s.bin", $target_path, $parsedid);
@mkdir($target_path, 0777, true);
if ($npar < 8 || $npar > 16 || $version != 256 || $disccount != 1) {
printf("bad file: version=%d, disccount=%d, total=%d, npar=%d, tool=%s",
$version, $disccount, $total, $npar, $tool);
return;
}
if (!@file_exists($target_file)) {
if(!move_uploaded_file($tmpname, $target_file)) {
echo "there was an error uploading the file, please try again!";
return;
}
printf("%s has been uploaded", $parsedid);
return;
}
$fp = fopen($tmpname, 'rb');
$fpstats = fstat($fp);
$db = parse_container_atom($fp, 0, $fpstats['size']);
foreach ($db as $entry) if($entry['name'] == 'CTDB') $ctdb = $entry;
if (@file_exists($target_file)) {
$fp1 = fopen($target_file, 'rb');
$fp1stats = fstat($fp1);
$db1 = parse_container_atom($fp1, 0, $fp1stats['size']);
foreach ($db1 as $entry) if($entry['name'] == 'CTDB') $ctdb1 = $entry;
}
$ftyp['name'] = 'ftyp';
$ftyp['value'] = 'CTDB';
$newctdb['name'] = 'CTDB';
$newhead['name'] = 'HEAD';
$newtotal['name'] = 'TOTL';
$newtotal['value'] = $ctdb1 ? BigEndian2String(BigEndian2Int(substr($ctdb1['HEAD']['value'],8,4)) + 1,4) : substr($ctdb['HEAD']['value'],8,4);
$newhead['subatoms'][] = $newtotal;
$newctdb['subatoms'][] = $newhead;
$discs = 0;
foreach ($ctdb['subatoms'] as $disc)
if ($disc['name'] == 'DISC')
{
$crc = $disc['CRC ']['value'];
$newdisc = false;
$newdisc['name'] = 'DISC';
$newdisc['subatoms'][] = $disc['CRC '];
$newdisc['subatoms'][] = $disc['NPAR'];
$newdisc['subatoms'][] = $disc['CONF'];
fseek($fp, $disc['PAR ']['offset']);
$newpar['name'] = 'PAR ';
$newpar['value'] = fread($fp, 16);
$newdisc['subatoms'][] = $newpar;
$newctdb['subatoms'][] = $newdisc;
$discs++;
}
if ($discs > 1)
die('One disc at a time, please');
if ($discs < 1)
die('No disc records found');
if ($ctdb1)
foreach ($ctdb1['subatoms'] as $disc)
if ($disc['name'] == 'DISC')
{
if ($crc == $disc['CRC ']['value'])
die("duplicate entry");
$newdisc = false;
$newdisc['name'] = 'DISC';
$newdisc['subatoms'][] = $disc['CRC '];
$newdisc['subatoms'][] = $disc['NPAR'];
$newdisc['subatoms'][] = $disc['CONF'];
fseek($fp1, $disc['PAR ']['offset']);
$newpar['name'] = 'PAR ';
$newpar['value'] = fread($fp1, 16);
$newdisc['subatoms'][] = $newpar;
$newctdb['subatoms'][] = $newdisc;
}
$destpath = sprintf("%s/%s/", $target_path, $parsedid);
@mkdir($destpath, 0777, true);
$tname = sprintf("%s/ctdb.tmp", $destpath);
$tfp = fopen($tname, 'wb');
unparse_atom($tfp,$ftyp);
unparse_atom($tfp,$newctdb);
fclose($tfp);
$crca = BigEndian2Int(substr($crc,0,2));
$crcb = BigEndian2Int(substr($crc,2,2));
$destname = sprintf("%s/%04x%04x.bin", $destpath, $crca, $crcb);
if(!move_uploaded_file($tmpname, $destname))
die('error uploading file');
if(!rename($tname,sprintf("%s/ctdb.bin", $destpath)))
die('error uploading file');
fclose($fp);
if ($fp1) fclose($fp1);
printf("%s has been updated", $parsedid);
?>

2
CUETools.Processor/Processor.cs
View File

@@ -2757,7 +2757,7 @@ string status = processor.Go();
CheckStop();
if (this.CUEToolsProgress == null)
return;
_progress.percentDisk = 1.0;
_progress.percentDisk = 0;
_progress.percentTrck = e.percent;
_progress.offset = 0;
_progress.status = e.uri;