// /*************************************************************************** // Aaru Data Preservation Suite // ---------------------------------------------------------------------------- // // Filename : Program.cs // Author(s) : Natalia Portillo // // Component : Aaru Server Task. // // --[ Description ] ---------------------------------------------------------- // // Runs time consuming server tasks. // // --[ License ] -------------------------------------------------------------- // // This library is free software; you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as // published by the Free Software Foundation; either version 2.1 of the // License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, see . // // ---------------------------------------------------------------------------- // Copyright © 2011-2024 Natalia Portillo // ****************************************************************************/ using System; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Security.Cryptography; using System.Text.Json; using Aaru.CommonTypes.Enums; using Aaru.Server.Models; using HtmlAgilityPack; using Microsoft.EntityFrameworkCore; namespace Aaru.Server.Task; class Program { public static void Main(string[] args) { DateTime start, end; var counter = 0; start = DateTime.UtcNow; System.Console.WriteLine("{0}: Connecting to database...", DateTime.UtcNow); var ctx = new AaruServerContext(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Migrating database to latest version...", DateTime.UtcNow); start = DateTime.UtcNow; ctx.Database.Migrate(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", DateTime.UtcNow, (end - start).TotalSeconds); WebClient client; try { System.Console.WriteLine("{0}: Retrieving USB IDs from Linux USB...", DateTime.UtcNow); start = DateTime.UtcNow; client = new WebClient(); var sr = new StringReader(client.DownloadString("http://www.linux-usb.org/usb.ids")); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); UsbVendor vendor = null; var newVendors = 0; var newProducts = 0; var modifiedVendors = 0; var modifiedProducts = 0; start = DateTime.UtcNow; System.Console.WriteLine("{0}: Adding and updating database entries...", DateTime.UtcNow); do { if(counter == 1000) { DateTime start2 = DateTime.UtcNow; System.Console.WriteLine("{0}: Saving changes", start2); ctx.SaveChanges(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start2).TotalSeconds); counter = 0; } string line = sr.ReadLine(); if(line is null) break; if(line.Length == 0 || line[0] == '#') continue; ushort number; string name; if(line[0] == '\t') { try { number = Convert.ToUInt16(line.Substring(1, 4), 16); } catch(FormatException) { continue; } if(number == 0) continue; name = line.Substring(7); UsbProduct product = ctx.UsbProducts.FirstOrDefault(p => p.ProductId == number && p.Vendor != null && p.Vendor.VendorId == vendor.VendorId); if(product is null) { product = new UsbProduct(vendor, number, name); ctx.UsbProducts.Add(product); System.Console.WriteLine("{0}: Will add product {1} with ID {2:X4} and vendor {3} ({4:X4})", DateTime.UtcNow, product.Product, product.ProductId, product.Vendor?.Vendor ?? "null", product.Vendor?.VendorId ?? 0); newProducts++; counter++; } else if(name != product.Product) { System.Console. WriteLine( "{0}: Will modify product with ID {1:X4} and vendor {2} ({3:X4}) from \"{4}\" to \"{5}\"", DateTime.UtcNow, product.ProductId, product.Vendor?.Vendor ?? "null", product.Vendor?.VendorId ?? 0, product.Product, name); product.Product = name; product.ModifiedWhen = DateTime.UtcNow; modifiedProducts++; counter++; } continue; } try { number = Convert.ToUInt16(line.Substring(0, 4), 16); } catch(FormatException) { continue; } if(number == 0) continue; name = line.Substring(6); vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == number); if(vendor is null) { vendor = new UsbVendor(number, name); ctx.UsbVendors.Add(vendor); System.Console.WriteLine("{0}: Will add vendor {1} with ID {2:X4}", DateTime.UtcNow, vendor.Vendor, vendor.VendorId); newVendors++; counter++; } else if(name != vendor.Vendor) { System.Console.WriteLine("{0}: Will modify vendor with ID {1:X4} from \"{2}\" to \"{3}\"", DateTime.UtcNow, vendor.VendorId, vendor.Vendor, name); vendor.Vendor = name; vendor.ModifiedWhen = DateTime.UtcNow; modifiedVendors++; counter++; } } while(true); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Saving database changes...", DateTime.UtcNow); start = DateTime.UtcNow; ctx.SaveChanges(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: {1} vendors added.", DateTime.UtcNow, newVendors); System.Console.WriteLine("{0}: {1} products added.", DateTime.UtcNow, newProducts); System.Console.WriteLine("{0}: {1} vendors modified.", DateTime.UtcNow, modifiedVendors); System.Console.WriteLine("{0}: {1} products modified.", DateTime.UtcNow, modifiedProducts); System.Console.WriteLine("{0}: Looking up a vendor", DateTime.UtcNow); start = DateTime.UtcNow; vendor = ctx.UsbVendors.FirstOrDefault(v => v.VendorId == 0x8086); if(vendor is null) System.Console.WriteLine("{0}: Error, could not find vendor.", DateTime.UtcNow); else System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow, vendor.Vendor); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Looking up a product", DateTime.UtcNow); start = DateTime.UtcNow; UsbProduct prd = ctx.UsbProducts.FirstOrDefault(p => p.ProductId == 0x0001 && p.Vendor.VendorId == 0x8086); if(prd is null) System.Console.WriteLine("{0}: Error, could not find product.", DateTime.UtcNow); else System.Console.WriteLine("{0}: Found {1}.", DateTime.UtcNow, prd.Product); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); } catch(Exception ex) { #if DEBUG if(Debugger.IsAttached) throw; #endif System.Console.WriteLine("{0}: Exception {1} filling USB IDs...", DateTime.UtcNow, ex); } System.Console.WriteLine("{0}: Fixing all devices without modification time...", DateTime.UtcNow); start = DateTime.UtcNow; foreach(Device device in ctx.Devices.Where(d => d.ModifiedWhen == null)) device.ModifiedWhen = device.AddedWhen; end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Committing changes...", DateTime.UtcNow); start = DateTime.UtcNow; ctx.SaveChanges(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); try { System.Console.WriteLine("{0}: Retrieving CompactDisc read offsets from AccurateRip...", DateTime.UtcNow); start = DateTime.UtcNow; client = new WebClient(); string html = client.DownloadString("http://www.accuraterip.com/driveoffsets.htm"); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); // The HTML is too malformed to process easily, so find start of table html = "" + html.Substring(html.IndexOf("
", StringComparison.Ordinal)); var doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode firstTable = doc.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]"); var firstRow = true; var addedOffsets = 0; var modifiedOffsets = 0; System.Console.WriteLine("{0}: Processing offsets...", DateTime.UtcNow); start = DateTime.UtcNow; foreach(HtmlNode row in firstTable.Descendants("tr")) { HtmlNode[] columns = row.Descendants("td").ToArray(); if(columns.Length != 4) { System.Console.WriteLine("{0}: Row does not have correct number of columns...", DateTime.UtcNow); continue; } string column0 = columns[0].InnerText; string column1 = columns[1].InnerText; string column2 = columns[2].InnerText; string column3 = columns[3].InnerText; if(firstRow) { if(column0.ToLowerInvariant() != "cd drive") { System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, columns[0].InnerText); break; } if(column1.ToLowerInvariant() != "correction offset") { System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, columns[1].InnerText); break; } if(column2.ToLowerInvariant() != "submitted by") { System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, columns[2].InnerText); break; } if(column3.ToLowerInvariant() != "percentage agree") { System.Console.WriteLine("{0}: Unexpected header \"{1}\" found...", DateTime.UtcNow, columns[3].InnerText); break; } firstRow = false; continue; } string manufacturer; string model; if(column0[0] == '-' && column0[1] == ' ') { manufacturer = null; model = column0.Substring(2).Trim(); } else { int cutOffset = column0.IndexOf(" - ", StringComparison.Ordinal); if(cutOffset == -1) { manufacturer = null; model = column0; } else { manufacturer = column0.Substring(0, cutOffset).Trim(); model = column0.Substring(cutOffset + 3).Trim(); } } switch(manufacturer) { case "Lite-ON": manufacturer = "JLMS"; break; case "LG Electronics": manufacturer = "HL-DT-ST"; break; case "Panasonic": manufacturer = "MATSHITA"; break; } CompactDiscOffset cdOffset = ctx.CdOffsets.FirstOrDefault(o => o.Manufacturer == manufacturer && o.Model == model); if(column1.ToLowerInvariant() == "[purged]") { if(cdOffset != null) ctx.CdOffsets.Remove(cdOffset); continue; } if(!short.TryParse(column1, out short offset)) continue; if(!int.TryParse(column2, out int submissions)) continue; if(column3[^1] != '%') continue; column3 = column3.Substring(0, column3.Length - 1); if(!float.TryParse(column3, out float percentage)) continue; percentage /= 100; if(cdOffset is null) { cdOffset = new CompactDiscOffset { AddedWhen = DateTime.UtcNow, ModifiedWhen = DateTime.UtcNow, Agreement = percentage, Manufacturer = manufacturer, Model = model, Offset = offset, Submissions = submissions }; ctx.CdOffsets.Add(cdOffset); addedOffsets++; } else { if(Math.Abs(cdOffset.Agreement - percentage) > 0) { cdOffset.Agreement = percentage; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(cdOffset.Offset != offset) { cdOffset.Offset = offset; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(cdOffset.Submissions != submissions) { cdOffset.Submissions = submissions; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(Math.Abs(cdOffset.Agreement - percentage) > 0 || cdOffset.Offset != offset || cdOffset.Submissions != submissions) modifiedOffsets++; } foreach(Device device in ctx.Devices. Where(d => d.Manufacturer == null && d.Model != null && d.Model.Trim() == model). Union(ctx.Devices.Where(d => d.Manufacturer != null && d.Manufacturer.Trim() == manufacturer && d.Model != null && d.Model == model))) { if(device.CdOffset == cdOffset && device.ModifiedWhen == cdOffset.ModifiedWhen) continue; device.CdOffset = cdOffset; device.ModifiedWhen = cdOffset.ModifiedWhen; } } end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); if(File.Exists("drive_offsets.json")) { var sr = new StreamReader("drive_offsets.json"); CompactDiscOffset[] offsets = JsonSerializer.Deserialize(sr.ReadToEnd()); if(offsets != null) { foreach(CompactDiscOffset offset in offsets) { CompactDiscOffset cdOffset = ctx.CdOffsets.FirstOrDefault(o => o.Manufacturer == offset.Manufacturer && o.Model == offset.Model); if(cdOffset is null) { offset.ModifiedWhen = DateTime.UtcNow; ctx.CdOffsets.Add(offset); addedOffsets++; } else { if(Math.Abs(cdOffset.Agreement - offset.Agreement) > 0 || offset.Agreement < 0) { cdOffset.Agreement = offset.Agreement; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(cdOffset.Offset != offset.Offset) { cdOffset.Offset = offset.Offset; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(cdOffset.Submissions != offset.Submissions) { cdOffset.Submissions = offset.Submissions; cdOffset.ModifiedWhen = DateTime.UtcNow; } if(Math.Abs(cdOffset.Agreement - offset.Agreement) > 0 || cdOffset.Offset != offset.Offset || cdOffset.Submissions != offset.Submissions) modifiedOffsets++; } } } } System.Console.WriteLine("{0}: Committing changes...", DateTime.UtcNow); start = DateTime.UtcNow; ctx.SaveChanges(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Added {1} offsets", end, addedOffsets); System.Console.WriteLine("{0}: Modified {1} offsets", end, modifiedOffsets); } catch(Exception ex) { #if DEBUG if(Debugger.IsAttached) throw; #endif System.Console.WriteLine("{0}: Exception {1} filling CompactDisc read offsets...", DateTime.UtcNow, ex); } if(!Directory.Exists("nes")) return; System.Console.WriteLine("{0}: Reading iNES/NES 2.0 headers...", DateTime.UtcNow); start = DateTime.UtcNow; var newHeaders = 0; var updatedHeaders = 0; counter = 0; foreach(string file in Directory.GetFiles("nes")) { try { var fs = new FileStream(file, FileMode.Open, FileAccess.Read); if(fs.Length <= 16) continue; var header = new byte[16]; var data = new byte[fs.Length - 16]; fs.Read(header, 0, 16); fs.Read(data, 0, data.Length); bool ines; bool nes20; ines = header[0] == 'N' && header[1] == 'E' && header[2] == 'S' && header[3] == 0x1A; nes20 = ines && (header[7] & 0x0C) == 0x08; if(!ines) continue; counter++; var info = new NesHeaderInfo { NametableMirroring = (header[6] & 0x1) == 0x1, BatteryPresent = (header[6] & 0x2) == 0x2, FourScreenMode = (header[6] & 0x8) == 0x8, Mapper = (ushort)(header[6] >> 4), ConsoleType = (NesConsoleType)(header[7] & 0x3) }; info.Mapper += (ushort)(header[7] & 0xF0); if(nes20) { info.Mapper += (ushort)((header[8] & 0xF) << 8); info.Submapper = (byte)(header[8] >> 4); info.TimingMode = (NesTimingMode)(header[12] & 0x3); switch(info.ConsoleType) { case NesConsoleType.Vs: info.VsPpuType = (NesVsPpuType)(header[13] & 0xF); info.VsHardwareType = (NesVsHardwareType)(header[13] >> 4); break; case NesConsoleType.Extended: info.ExtendedConsoleType = (NesExtendedConsoleType)(header[13] & 0xF); break; } info.DefaultExpansionDevice = (NesDefaultExpansionDevice)header[15]; } var hasher = SHA256.Create(); byte[] hashBytes = hasher.ComputeHash(data); var hashChars = new char[64]; for(var i = 0; i < 32; i++) { int a = hashBytes[i] >> 4; int b = hashBytes[i] & 0xF; hashChars[i * 2] = a > 9 ? (char)(a + 0x57) : (char)(a + 0x30); hashChars[i * 2 + 1] = b > 9 ? (char)(b + 0x57) : (char)(b + 0x30); } info.Sha256 = new string(hashChars); NesHeaderInfo existing = ctx.NesHeaders.FirstOrDefault(h => h.Sha256 == info.Sha256); if(existing == null) { info.AddedWhen = DateTime.UtcNow; info.ModifiedWhen = info.AddedWhen; ctx.NesHeaders.Add(info); newHeaders++; continue; } var modified = false; if(existing.NametableMirroring != info.NametableMirroring) { existing.NametableMirroring = info.NametableMirroring; modified = true; } if(existing.BatteryPresent != info.BatteryPresent) { existing.BatteryPresent = info.BatteryPresent; modified = true; } if(existing.FourScreenMode != info.FourScreenMode) { existing.FourScreenMode = info.FourScreenMode; modified = true; } if(existing.Mapper != info.Mapper) { existing.Mapper = info.Mapper; modified = true; } if(existing.ConsoleType != info.ConsoleType) { existing.ConsoleType = info.ConsoleType; modified = true; } if(existing.Submapper != info.Submapper) { existing.Submapper = info.Submapper; modified = true; } if(existing.TimingMode != info.TimingMode) { existing.TimingMode = info.TimingMode; modified = true; } if(existing.VsPpuType != info.VsPpuType) { existing.VsPpuType = info.VsPpuType; modified = true; } if(existing.VsHardwareType != info.VsHardwareType) { existing.VsHardwareType = info.VsHardwareType; modified = true; } if(existing.ExtendedConsoleType != info.ExtendedConsoleType) { existing.ExtendedConsoleType = info.ExtendedConsoleType; modified = true; } if(existing.DefaultExpansionDevice != info.DefaultExpansionDevice) { existing.DefaultExpansionDevice = info.DefaultExpansionDevice; modified = true; } if(!modified) continue; existing.ModifiedWhen = DateTime.UtcNow; updatedHeaders++; } catch(Exception ex) { #if DEBUG if(Debugger.IsAttached) throw; #endif System.Console.WriteLine("{0}: Exception {1} with file {2}...", DateTime.UtcNow, ex, file); } } end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); System.Console.WriteLine("{0}: Processed {1} iNES/NES 2.0 headers...", DateTime.UtcNow, counter); System.Console.WriteLine("{0}: Added {1} iNES/NES 2.0 headers...", DateTime.UtcNow, newHeaders); System.Console.WriteLine("{0}: Updated {1} iNES/NES 2.0 headers...", DateTime.UtcNow, updatedHeaders); System.Console.WriteLine("{0}: Committing changes...", DateTime.UtcNow); start = DateTime.UtcNow; ctx.SaveChanges(); end = DateTime.UtcNow; System.Console.WriteLine("{0}: Took {1:F2} seconds", end, (end - start).TotalSeconds); } }