aaru-dps/Aaru.Server
1
0
Fork 0
mirror of https://github.com/aaru-dps/Aaru.Server.git synced 2025-12-16 19:24:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor task.

Browse Source
This commit is contained in:
Natalia Portillo
2019-11-02 21:39:19 +00:00
parent 49823cc89d
commit 595f2bd6fd
1 changed files with 137 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
DiscImageChef.Server.Task/Program.cs
View File

@@ -41,7 +41,7 @@ using Microsoft.EntityFrameworkCore;
namespace DiscImageChef.Server.Task namespace DiscImageChef.Server.Task
{ {
class Program internal class Program
{ {
public static void Main(string[] args) public static void Main(string[] args)
{ {
@@ -49,7 +49,7 @@ namespace DiscImageChef.Server.Task
start = DateTime.UtcNow; start = DateTime.UtcNow;
System.Console.WriteLine("{0}: Connecting to database...", DateTime.UtcNow); System.Console.WriteLine("{0}: Connecting to database...", DateTime.UtcNow);
DicServerContext ctx = new DicServerContext(); var ctx = new DicServerContext();
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
@@ -66,24 +66,24 @@ namespace DiscImageChef.Server.Task
System.Console.WriteLine("{0}: Retrieving USB IDs from Linux USB...", DateTime.UtcNow); System.Console.WriteLine("{0}: Retrieving USB IDs from Linux USB...", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
client = new WebClient(); client = new WebClient();
StringReader sr = new StringReader(client.DownloadString("http://www.linux-usb.org/usb.ids")); var sr = new StringReader(client.DownloadString("http://www.linux-usb.org/usb.ids"));
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
UsbVendor vendor = null; UsbVendor vendor = null;
int newVendors = 0; var newVendors = 0;
int newProducts = 0; var newProducts = 0;
int modifiedVendors = 0; var modifiedVendors = 0;
int modifiedProducts = 0; var modifiedProducts = 0;
int counter = 0; var counter = 0;
start = DateTime.UtcNow; start = DateTime.UtcNow;
System.Console.WriteLine("{0}: Adding and updating database entries...", DateTime.UtcNow); System.Console.WriteLine("{0}: Adding and updating database entries...", DateTime.UtcNow);
do do
{ {
if(counter == 1000) if (counter == 1000)
{ {
DateTime start2 = DateTime.UtcNow; var start2 = DateTime.UtcNow;
System.Console.WriteLine("{0}: Saving changes", start2); System.Console.WriteLine("{0}: Saving changes", start2);
ctx.SaveChanges(); ctx.SaveChanges();
end = DateTime.UtcNow; end = DateTime.UtcNow;
@@ -91,28 +91,34 @@ namespace DiscImageChef.Server.Task
counter = 0; counter = 0;
} }
string line = sr.ReadLine(); var line = sr.ReadLine();
if(line is null) break; if (line is null) break;
if(line.Length == 0 || line[0] == '#') continue; if (line.Length == 0 || line[0] == '#') continue;
ushort number; ushort number;
string name; string name;
if(line[0] == '\t') if (line[0] == '\t')
{ {
try { number = Convert.ToUInt16(line.Substring(1, 4), 16); } try
catch(FormatException) { continue; } {
number = Convert.ToUInt16(line.Substring(1, 4), 16);
}
catch (FormatException)
{
continue;
}
if(number == 0) continue; if (number == 0) continue;
name = line.Substring(7); name = line.Substring(7);
UsbProduct product = var product =
ctx.UsbProducts.FirstOrDefault(p => p.ProductId == number && p.Vendor != null && ctx.UsbProducts.FirstOrDefault(p => p.ProductId == number && p.Vendor != null &&
p.Vendor.VendorId == vendor.VendorId); p.Vendor.VendorId == vendor.VendorId);
if(product is null) if (product is null)
{ {
product = new UsbProduct(vendor, number, name); product = new UsbProduct(vendor, number, name);
ctx.UsbProducts.Add(product); ctx.UsbProducts.Add(product);
@@ -122,10 +128,11 @@ namespace DiscImageChef.Server.Task
newProducts++; newProducts++;
counter++; counter++;
} }
else if(name != product.Product) else if (name != product.Product)
{ {
System.Console System.Console
.WriteLine("{0}: Will modify product with ID {1:X4} and vendor {2} ({3:X4}) from \"{4}\" to \"{5}\"", .WriteLine(
"{0}: Will modify product with ID {1:X4} and vendor {2} ({3:X4}) from \"{4}\" to \"{5}\"",
DateTime.UtcNow, product.ProductId, product.Vendor?.Vendor ?? "null", DateTime.UtcNow, product.ProductId, product.Vendor?.Vendor ?? "null",
product.Vendor?.VendorId ?? 0, product.Vendor?.VendorId ?? 0,
product.Product, name); product.Product, name);
@@ -138,25 +145,32 @@ namespace DiscImageChef.Server.Task
continue; continue;
} }
try { number = Convert.ToUInt16(line.Substring(0, 4), 16); } try
catch(FormatException) { continue; } {
number = Convert.ToUInt16(line.Substring(0, 4), 16);
}
catch (FormatException)
{
continue;
}
if(number == 0) continue; if (number == 0) continue;
name = line.Substring(6); name = line.Substring(6);
vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == number); vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == number);
if(vendor is null) if (vendor is null)
{ {
vendor = new UsbVendor(number, name); vendor = new UsbVendor(number, name);
ctx.UsbVendors.Add(vendor); ctx.UsbVendors.Add(vendor);
System.Console.WriteLine("{0}: Will add vendor {1} with ID {2:X4}", DateTime.UtcNow, vendor.Vendor, System.Console.WriteLine("{0}: Will add vendor {1} with ID {2:X4}", DateTime.UtcNow,
vendor.Vendor,
vendor.VendorId); vendor.VendorId);
newVendors++; newVendors++;
counter++; counter++;
} }
else if(name != vendor.Vendor) else if (name != vendor.Vendor)
{ {
System.Console.WriteLine("{0}: Will modify vendor with ID {1:X4} from \"{2}\" to \"{3}\"", System.Console.WriteLine("{0}: Will modify vendor with ID {1:X4} from \"{2}\" to \"{3}\"",
DateTime.UtcNow, vendor.VendorId, vendor.Vendor, name); DateTime.UtcNow, vendor.VendorId, vendor.Vendor, name);
@@ -165,8 +179,7 @@ namespace DiscImageChef.Server.Task
modifiedVendors++; modifiedVendors++;
counter++; counter++;
} }
} } while (true);
while(true);
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
@@ -185,7 +198,7 @@ namespace DiscImageChef.Server.Task
System.Console.WriteLine("{0}: Looking up a vendor", DateTime.UtcNow); System.Console.WriteLine("{0}: Looking up a vendor", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == 0x8086); vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == 0x8086);
if(vendor is null) System.Console.WriteLine("{0}: Error, could not find vendor.", DateTime.UtcNow); if (vendor is null) System.Console.WriteLine("{0}: Error, could not find vendor.", DateTime.UtcNow);
else else
System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow, System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow,
vendor.Vendor); vendor.Vendor);
@@ -194,24 +207,24 @@ namespace DiscImageChef.Server.Task
System.Console.WriteLine("{0}: Looking up a product", DateTime.UtcNow); System.Console.WriteLine("{0}: Looking up a product", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
UsbProduct prd = var prd =
ctx.UsbProducts.FirstOrDefault(p => p.ProductId == 0x0001 && p.Vendor.VendorId == 0x8086); ctx.UsbProducts.FirstOrDefault(p => p.ProductId == 0x0001 && p.Vendor.VendorId == 0x8086);
if(prd is null) System.Console.WriteLine("{0}: Error, could not find product.", DateTime.UtcNow); if (prd is null) System.Console.WriteLine("{0}: Error, could not find product.", DateTime.UtcNow);
else System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow, prd.Product); else System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow, prd.Product);
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
} }
catch(Exception ex) catch (Exception ex)
{ {
#if DEBUG #if DEBUG
if(Debugger.IsAttached) throw; if (Debugger.IsAttached) throw;
#endif #endif
System.Console.WriteLine("{0}: Exception {1} filling USB IDs...", DateTime.UtcNow, ex); System.Console.WriteLine("{0}: Exception {1} filling USB IDs...", DateTime.UtcNow, ex);
} }
System.Console.WriteLine("{0}: Fixing all devices without modification time...", DateTime.UtcNow); System.Console.WriteLine("{0}: Fixing all devices without modification time...", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
foreach(Device device in ctx.Devices.Where(d => d.ModifiedWhen == null)) foreach (var device in ctx.Devices.Where(d => d.ModifiedWhen == null))
device.ModifiedWhen = device.AddedWhen; device.ModifiedWhen = device.AddedWhen;
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
@@ -224,11 +237,12 @@ namespace DiscImageChef.Server.Task
try try
{ {
System.Console.WriteLine("{0}: Retrieving CompactDisc read offsets from AccurateRip...", DateTime.UtcNow); System.Console.WriteLine("{0}: Retrieving CompactDisc read offsets from AccurateRip...",
DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
client = new WebClient(); client = new WebClient();
string html = client.DownloadString("http://www.accuraterip.com/driveoffsets.htm"); var html = client.DownloadString("http://www.accuraterip.com/driveoffsets.htm");
end = DateTime.UtcNow; end = DateTime.UtcNow;
System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds);
@@ -236,56 +250,57 @@ namespace DiscImageChef.Server.Task
html = "<html><body><table><tr>" + html = "<html><body><table><tr>" +
html.Substring(html.IndexOf("<td bgcolor=\"#000000\">", StringComparison.Ordinal)); html.Substring(html.IndexOf("<td bgcolor=\"#000000\">", StringComparison.Ordinal));
HtmlDocument doc = new HtmlDocument(); var doc = new HtmlDocument();
doc.LoadHtml(html); doc.LoadHtml(html);
HtmlNode firstTable = doc.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]"); var firstTable = doc.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]");
bool firstRow = true; var firstRow = true;
int addedOffsets = 0; var addedOffsets = 0;
int modifiedOffsets = 0; var modifiedOffsets = 0;
System.Console.WriteLine("{0}: Processing offsets...", DateTime.UtcNow); System.Console.WriteLine("{0}: Processing offsets...", DateTime.UtcNow);
start = DateTime.UtcNow; start = DateTime.UtcNow;
foreach(HtmlNode row in firstTable.Descendants("tr")) foreach (var row in firstTable.Descendants("tr"))
{ {
HtmlNode[] columns = row.Descendants("td").ToArray(); var columns = row.Descendants("td").ToArray();
if(columns.Length != 4) if (columns.Length != 4)
{ {
System.Console.WriteLine("{0}: Row does not have correct number of columns...", DateTime.UtcNow); System.Console.WriteLine("{0}: Row does not have correct number of columns...",
DateTime.UtcNow);
continue; continue;
} }
string column0 = columns[0].InnerText; var column0 = columns[0].InnerText;
string column1 = columns[1].InnerText; var column1 = columns[1].InnerText;
string column2 = columns[2].InnerText; var column2 = columns[2].InnerText;
string column3 = columns[3].InnerText; var column3 = columns[3].InnerText;
if(firstRow) if (firstRow)
{ {
if(column0.ToLowerInvariant() != "cd drive") if (column0.ToLowerInvariant() != "cd drive")
{ {
System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[0].InnerText); columns[0].InnerText);
break; break;
} }
if(column1.ToLowerInvariant() != "correction offset") if (column1.ToLowerInvariant() != "correction offset")
{ {
System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[1].InnerText); columns[1].InnerText);
break; break;
} }
if(column2.ToLowerInvariant() != "submitted by") if (column2.ToLowerInvariant() != "submitted by")
{ {
System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[2].InnerText); columns[2].InnerText);
break; break;
} }
if(column3.ToLowerInvariant() != "percentage agree") if (column3.ToLowerInvariant() != "percentage agree")
{ {
System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow,
columns[3].InnerText); columns[3].InnerText);
@@ -299,16 +314,16 @@ namespace DiscImageChef.Server.Task
string manufacturer; string manufacturer;
string model; string model;
if(column0[0] == '-' && column0[1] == ' ') if (column0[0] == '-' && column0[1] == ' ')
{ {
manufacturer = null; manufacturer = null;
model = column0.Substring(2).Trim(); model = column0.Substring(2).Trim();
} }
else else
{ {
int cutOffset = column0.IndexOf(" - ", StringComparison.Ordinal); var cutOffset = column0.IndexOf(" - ", StringComparison.Ordinal);
if(cutOffset == -1) if (cutOffset == -1)
{ {
manufacturer = null; manufacturer = null;
model = column0; model = column0;
@@ -320,7 +335,7 @@ namespace DiscImageChef.Server.Task
} }
} }
switch(manufacturer) switch (manufacturer)
{ {
case "Lite-ON": case "Lite-ON":
manufacturer = "JLMS"; manufacturer = "JLMS";
@@ -333,27 +348,27 @@ namespace DiscImageChef.Server.Task
break; break;
} }
CompactDiscOffset cdOffset = var cdOffset =
ctx.CdOffsets.FirstOrDefault(o => o.Manufacturer == manufacturer && o.Model == model); ctx.CdOffsets.FirstOrDefault(o => o.Manufacturer == manufacturer && o.Model == model);
if(column1.ToLowerInvariant() == "[purged]") if (column1.ToLowerInvariant() == "[purged]")
{ {
if(cdOffset != null) ctx.CdOffsets.Remove(cdOffset); if (cdOffset != null) ctx.CdOffsets.Remove(cdOffset);
continue; continue;
} }
if(!short.TryParse(column1, out short offset)) continue; if (!short.TryParse(column1, out var offset)) continue;
if(!int.TryParse(column2, out int submissions)) continue; if (!int.TryParse(column2, out var submissions)) continue;
if(column3[column3.Length - 1] != '%') continue; if (column3[column3.Length - 1] != '%') continue;
column3 = column3.Substring(0, column3.Length - 1); column3 = column3.Substring(0, column3.Length - 1);
if(!float.TryParse(column3, out float percentage)) continue; if (!float.TryParse(column3, out var percentage)) continue;
percentage /= 100; percentage /= 100;
if(cdOffset is null) if (cdOffset is null)
{ {
cdOffset = new CompactDiscOffset cdOffset = new CompactDiscOffset
{ {
@@ -371,29 +386,29 @@ namespace DiscImageChef.Server.Task
} }
else else
{ {
if(Math.Abs(cdOffset.Agreement - percentage) > 0) if (Math.Abs(cdOffset.Agreement - percentage) > 0)
{ {
cdOffset.Agreement = percentage; cdOffset.Agreement = percentage;
cdOffset.ModifiedWhen = DateTime.UtcNow; cdOffset.ModifiedWhen = DateTime.UtcNow;
} }
if(cdOffset.Offset != offset) if (cdOffset.Offset != offset)
{ {
cdOffset.Offset = offset; cdOffset.Offset = offset;
cdOffset.ModifiedWhen = DateTime.UtcNow; cdOffset.ModifiedWhen = DateTime.UtcNow;
} }
if(cdOffset.Submissions != submissions) if (cdOffset.Submissions != submissions)
{ {
cdOffset.Submissions = submissions; cdOffset.Submissions = submissions;
cdOffset.ModifiedWhen = DateTime.UtcNow; cdOffset.ModifiedWhen = DateTime.UtcNow;
} }
if(Math.Abs(cdOffset.Agreement - percentage) > 0 || cdOffset.Offset != offset || if (Math.Abs(cdOffset.Agreement - percentage) > 0 || cdOffset.Offset != offset ||
cdOffset.Submissions != submissions) modifiedOffsets++; cdOffset.Submissions != submissions) modifiedOffsets++;
} }
foreach(Device device in ctx foreach (var device in ctx
.Devices .Devices
.Where(d => d.Manufacturer == null && d.Model != null && .Where(d => d.Manufacturer == null && d.Model != null &&
d.Model.Trim() == model) d.Model.Trim() == model)
@@ -402,7 +417,7 @@ namespace DiscImageChef.Server.Task
d.Model != null && d.Model != null &&
d.Model == model))) d.Model == model)))
{ {
if(device.CdOffset == cdOffset && device.ModifiedWhen == cdOffset.ModifiedWhen) continue; if (device.CdOffset == cdOffset && device.ModifiedWhen == cdOffset.ModifiedWhen) continue;
device.CdOffset = cdOffset; device.CdOffset = cdOffset;
device.ModifiedWhen = cdOffset.ModifiedWhen; device.ModifiedWhen = cdOffset.ModifiedWhen;
@@ -421,11 +436,11 @@ namespace DiscImageChef.Server.Task
System.Console.WriteLine("{0}: Added {1} offsets", end, addedOffsets); System.Console.WriteLine("{0}: Added {1} offsets", end, addedOffsets);
System.Console.WriteLine("{0}: Modified {1} offsets", end, modifiedOffsets); System.Console.WriteLine("{0}: Modified {1} offsets", end, modifiedOffsets);
} }
catch(Exception ex) catch (Exception ex)
{ {
#if DEBUG #if DEBUG
if(Debugger.IsAttached) throw; if (Debugger.IsAttached) throw;
#endif #endif
System.Console.WriteLine("{0}: Exception {1} filling CompactDisc read offsets...", DateTime.UtcNow, ex); System.Console.WriteLine("{0}: Exception {1} filling CompactDisc read offsets...", DateTime.UtcNow, ex);
} }
} }