aaru-dps/Aaru
1
0
Fork 0
mirror of https://github.com/aaru-dps/Aaru.git synced 2025-12-16 19:24:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fill CompactDisc read offsets in database from AccurateRip list.

Browse Source
This commit is contained in:
Natalia Portillo
2018-12-25 13:27:39 +00:00
parent f6c2b087a2
commit 17d6c0080f
2 changed files with 224 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
DiscImageChef.Server.Task/DiscImageChef.Server.Task.csproj
View File

@@ -32,6 +32,12 @@
<WarningLevel>4</WarningLevel> <WarningLevel>4</WarningLevel>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="DiscImageChef.CommonTypes, Version=4.5.99.1693, Culture=neutral, PublicKeyToken=null">
<HintPath>..\..\..\..\..\tmp\claunia-temp-aspnet-0\e277a754\assembly\shadow\34ef13c2\840cc539_b0e3d6fb_00000001\DiscImageChef.CommonTypes.dll</HintPath>
</Reference>
<Reference Include="DiscImageChef.Decoders, Version=4.5.99.1693, Culture=neutral, PublicKeyToken=null">
<HintPath>..\..\..\..\..\tmp\claunia-temp-aspnet-0\e277a754\assembly\shadow\3a88f867\8a6b2e9c_b0e3d6fb_00000001\DiscImageChef.Decoders.dll</HintPath>
</Reference>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<HintPath>..\..\..\.nuget\packages\entityframework\6.2.0\lib\net45\EntityFramework.dll</HintPath> <HintPath>..\..\..\.nuget\packages\entityframework\6.2.0\lib\net45\EntityFramework.dll</HintPath>
</Reference> </Reference>
@@ -45,6 +51,10 @@
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\DiscImageChef.Helpers\DiscImageChef.Helpers.csproj">
<Project>{f8bdf57b-1571-4cd0-84b3-b422088d359a}</Project>
<Name>DiscImageChef.Helpers</Name>
</ProjectReference>
<ProjectReference Include="..\DiscImageChef.Server\DiscImageChef.Server.csproj"> <ProjectReference Include="..\DiscImageChef.Server\DiscImageChef.Server.csproj">
<Project>{911addf5-e5fa-445d-88cd-c7f8ffdbe645}</Project> <Project>{911addf5-e5fa-445d-88cd-c7f8ffdbe645}</Project>
<Name>DiscImageChef.Server</Name> <Name>DiscImageChef.Server</Name>
@@ -52,6 +62,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="EntityFramework" Version="6.2.0" /> <PackageReference Include="EntityFramework" Version="6.2.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.8.11" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="App.config" /> <None Include="App.config" />

212
DiscImageChef.Server.Task/Program.cs
View File

@@ -6,6 +6,7 @@ using System.Linq;
using System.Net; using System.Net;
using DiscImageChef.Server.Migrations; using DiscImageChef.Server.Migrations;
using DiscImageChef.Server.Models; using DiscImageChef.Server.Models;
using HtmlAgilityPack;
namespace DiscImageChef.Server.Task namespace DiscImageChef.Server.Task
{ {
@@ -28,11 +29,13 @@ namespace DiscImageChef.Server.Task
end = DateTime.UtcNow; end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
WebClient client;
try try
{ {
Console.WriteLine("{0}: Retrieving USB IDs from Linux USB...", DateTime.UtcNow); Console.WriteLine("{0}: Retrieving USB IDs from Linux USB...", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
WebClient client = new WebClient(); client = new WebClient();
StringReader sr = new StringReader(client.DownloadString("http://www.linux-usb.org/usb.ids")); StringReader sr = new StringReader(client.DownloadString("http://www.linux-usb.org/usb.ids"));
end = DateTime.UtcNow; end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
@@ -188,6 +191,213 @@ namespace DiscImageChef.Server.Task
ctx.SaveChanges(); ctx.SaveChanges();
end = DateTime.UtcNow; end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
try
{
Console.WriteLine("{0}: Retrieving CompactDisc read offsets from AccurateRip...", DateTime.UtcNow);
start = DateTime.UtcNow;
client = new WebClient();
string html = client.DownloadString("http://www.accuraterip.com/driveoffsets.htm");
end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
// The HTML is too malformed to process easily, so find start of table
html = "<html><body><table><tr>" +
html.Substring(html.IndexOf("<td bgcolor=\"#000000\">", StringComparison.Ordinal));
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlNode firstTable = doc.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]");
bool firstRow = true;
int addedOffsets = 0;
int modifiedOffsets = 0;
Console.WriteLine("{0}: Processing offsets...", DateTime.UtcNow);
start = DateTime.UtcNow;
foreach(HtmlNode row in firstTable.Descendants("tr"))
{
HtmlNode[] columns = row.Descendants("td").ToArray();
if(columns.Length != 4)
{
Console.WriteLine("{0}: Row does not have correct number of columns...", DateTime.UtcNow);
continue;
}
string column0 = columns[0].InnerText;
string column1 = columns[1].InnerText;
string column2 = columns[2].InnerText;
string column3 = columns[3].InnerText;
if(firstRow)
{
if(column0.ToLowerInvariant() != "cd drive")
{
Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[0].InnerText);
break;
}
if(column1.ToLowerInvariant() != "correction offset")
{
Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[1].InnerText);
break;
}
if(column2.ToLowerInvariant() != "submitted by")
{
Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[2].InnerText);
break;
}
if(column3.ToLowerInvariant() != "percentage agree")
{
Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[3].InnerText);
break;
}
firstRow = false;
continue;
}
string manufacturer;
string model;
if(column0[0] == '-' && column0[1] == ' ')
{
manufacturer = null;
model = column0.Substring(2).Trim();
}
else
{
int cutOffset = column0.IndexOf(" - ", StringComparison.Ordinal);
if(cutOffset == -1)
{
manufacturer = null;
model = column0;
}
else
{
manufacturer = column0.Substring(0, cutOffset).Trim();
model = column0.Substring(cutOffset + 3).Trim();
}
}
switch(manufacturer)
{
case "Lite-ON":
manufacturer = "JLMS";
break;
case "LG Electronics":
manufacturer = "HL-DT-ST";
break;
case "Panasonic":
manufacturer = "MATSHITA";
break;
}
CompactDiscOffset cdOffset =
ctx.CdOffsets.FirstOrDefault(o => o.Manufacturer == manufacturer && o.Model == model);
if(column1.ToLowerInvariant() == "[purged]")
{
if(cdOffset != null) ctx.CdOffsets.Remove(cdOffset);
continue;
}
if(!short.TryParse(column1, out short offset)) continue;
if(!int.TryParse(column2, out int submissions)) continue;
if(column3[column3.Length - 1] != '%') continue;
column3 = column3.Substring(0, column3.Length - 1);
if(!float.TryParse(column3, out float percentage)) continue;
percentage /= 100;
if(cdOffset is null)
{
cdOffset = new CompactDiscOffset
{
AddedWhen = DateTime.UtcNow,
ModifiedWhen = DateTime.UtcNow,
Agreement = percentage,
Manufacturer = manufacturer,
Model = model,
Offset = offset,
Submissions = submissions
};
ctx.CdOffsets.Add(cdOffset);
addedOffsets++;
}
else
{
if(Math.Abs(cdOffset.Agreement - percentage) > 0)
{
cdOffset.Agreement = percentage;
cdOffset.ModifiedWhen = DateTime.UtcNow;
}
if(cdOffset.Offset != offset)
{
cdOffset.Offset = offset;
cdOffset.ModifiedWhen = DateTime.UtcNow;
}
if(cdOffset.Submissions != submissions)
{
cdOffset.Submissions = submissions;
cdOffset.ModifiedWhen = DateTime.UtcNow;
}
if(Math.Abs(cdOffset.Agreement - percentage) > 0 || cdOffset.Offset != offset ||
cdOffset.Submissions != submissions) modifiedOffsets++;
}
foreach(Device device in ctx
.Devices
.Where(d => d.Manufacturer == null && d.Model != null &&
d.Model.Trim() == model)
.Union(ctx.Devices.Where(d => d.Manufacturer != null &&
d.Manufacturer.Trim() == manufacturer &&
d.Model != null &&
d.Model == model)))
{
if(device.CdOffset == cdOffset && device.ModifiedWhen == cdOffset.ModifiedWhen) continue;
device.CdOffset = cdOffset;
device.ModifiedWhen = cdOffset.ModifiedWhen;
}
}
end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
Console.WriteLine("{0}: Committing changes...", DateTime.UtcNow);
start = DateTime.UtcNow;
ctx.SaveChanges();
end = DateTime.UtcNow;
Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
Console.WriteLine("{0}: Added {1} offsets", end, addedOffsets);
Console.WriteLine("{0}: Modified {1} offsets", end, modifiedOffsets);
}
catch(Exception ex)
{
#if DEBUG
if(Debugger.IsAttached) throw;
#endif
Console.WriteLine("{0}: Exception {1} filling CompactDisc read offsets...", DateTime.UtcNow, ex);
}
} }
} }
} }