// /*************************************************************************** // Aaru Data Preservation Suite // ---------------------------------------------------------------------------- // // Filename : Remote.cs // Author(s) : Natalia Portillo // // Component : Core algorithms. // // --[ Description ] ---------------------------------------------------------- // // Handles connections to Aaru.Server. // // --[ License ] -------------------------------------------------------------- // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // ---------------------------------------------------------------------------- // Copyright © 2011-2021 Natalia Portillo // ****************************************************************************/ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Threading; using Aaru.CommonTypes.Metadata; using Aaru.Console; using Aaru.Database; using Aaru.Database.Models; using Aaru.Dto; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using Spectre.Console; using CdOffset = Aaru.Database.Models.CdOffset; using Version = Aaru.CommonTypes.Metadata.Version; namespace Aaru.Core { /// Handles connections to Aaru.Server public static class Remote { /// Submits a device report /// Device report public static void SubmitReport(DeviceReportV2 report) { var submitThread = new Thread(() => { Spectre.ProgressSingleSpinner(ctx => { ctx.AddTask("Uploading device report").IsIndeterminate(); try { string json = JsonConvert.SerializeObject(report, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); byte[] jsonBytes = Encoding.UTF8.GetBytes(json); var request = WebRequest.Create("https://www.aaru.app/api/uploadreportv2"); ((HttpWebRequest)request).UserAgent = $"Aaru {typeof(Version).Assembly.GetName().Version}"; request.Method = "POST"; request.ContentLength = jsonBytes.Length; request.ContentType = "application/json"; Stream reqStream = request.GetRequestStream(); reqStream.Write(jsonBytes, 0, jsonBytes.Length); reqStream.Close(); WebResponse response = request.GetResponse(); if(((HttpWebResponse)response).StatusCode != HttpStatusCode.OK) return; Stream data = response.GetResponseStream(); var reader = new StreamReader(data ?? throw new InvalidOperationException()); reader.ReadToEnd(); data.Close(); response.Close(); } catch(WebException) { // Can't connect to the server, do nothing } // ReSharper disable once RedundantCatchClause catch { #if DEBUG if(Debugger.IsAttached) throw; #endif } }); }); submitThread.Start(); } /// Updates the main database /// If true creates the database from scratch, otherwise updates an existing database public static void UpdateMainDatabase(bool create) { var mctx = AaruContext.Create(Settings.Settings.MainDbPath); if(create) { mctx.Database.EnsureCreated(); mctx.Database. ExecuteSqlRaw("CREATE TABLE IF NOT EXISTS \"__EFMigrationsHistory\" (\"MigrationId\" TEXT PRIMARY KEY, \"ProductVersion\" TEXT)"); foreach(string migration in mctx.Database.GetPendingMigrations()) { mctx.Database. ExecuteSqlRaw($"INSERT INTO \"__EFMigrationsHistory\" (MigrationId, ProductVersion) VALUES ('{migration}', '0.0.0')"); } } else mctx.Database.Migrate(); mctx.SaveChanges(); try { long lastUpdate = 0; DateTime latest = DateTime.MinValue; if(!create) { List latestAll = new(); if(mctx.UsbVendors.Any()) latestAll.Add(mctx.UsbVendors.Max(v => v.ModifiedWhen)); if(mctx.UsbProducts.Any()) latestAll.Add(mctx.UsbProducts.Max(p => p.ModifiedWhen)); if(mctx.CdOffsets.Any()) latestAll.Add(mctx.CdOffsets.Max(o => o.ModifiedWhen)); if(mctx.Devices.Any()) latestAll.Add(mctx.Devices.Max(d => d.LastSynchronized)); if(latestAll.Any()) { latest = latestAll.Max(t => t); lastUpdate = (latest.ToFileTimeUtc() - new DateTime(1970, 1, 1).ToFileTimeUtc()) / 10000000; } } if(lastUpdate == 0) { create = true; AaruConsole.WriteLine("Creating main database"); } else { AaruConsole.WriteLine("Updating main database"); AaruConsole.WriteLine("Last update: {0}", latest); } DateTime updateStart = DateTime.UtcNow; var request = WebRequest.Create($"https://www.aaru.app/api/update?timestamp={lastUpdate}"); ((HttpWebRequest)request).UserAgent = $"Aaru {typeof(Version).Assembly.GetName().Version}"; request.Method = "GET"; request.ContentType = "application/json"; WebResponse response = request.GetResponse(); if(((HttpWebResponse)response).StatusCode != HttpStatusCode.OK) { AaruConsole.ErrorWriteLine("Error {0} when trying to get updated entities.", ((HttpWebResponse)response).StatusCode); return; } Stream data = response.GetResponseStream(); var reader = new StreamReader(data ?? throw new InvalidOperationException()); SyncDto sync = JsonConvert.DeserializeObject(reader.ReadToEnd()) ?? new SyncDto(); if(create) { AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Adding USB vendors"); task.MaxValue = sync.UsbVendors.Count; foreach(UsbVendorDto vendor in sync.UsbVendors) { task.Increment(1); mctx.UsbVendors.Add(new UsbVendor(vendor.VendorId, vendor.Vendor)); } }); AaruConsole.WriteLine("Added {0} usb vendors", sync.UsbVendors.Count); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Adding USB products"); task.MaxValue = sync.UsbProducts.Count; foreach(UsbProductDto product in sync.UsbProducts) { task.Increment(1); mctx.UsbProducts.Add(new UsbProduct(product.VendorId, product.ProductId, product.Product)); } }); AaruConsole.WriteLine("Added {0} usb products", sync.UsbProducts.Count); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Adding CompactDisc read offsets"); task.MaxValue = sync.Offsets.Count; foreach(CdOffsetDto offset in sync.Offsets) { task.Increment(1); mctx.CdOffsets.Add(new CdOffset(offset) { Id = offset.Id }); } }); AaruConsole.WriteLine("Added {0} CompactDisc read offsets", sync.Offsets.Count); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Adding known devices"); task.MaxValue = sync.Devices.Count; foreach(DeviceDto device in sync.Devices) { task.Increment(1); mctx.Devices.Add(new Device(device) { Id = device.Id }); } }); AaruConsole.WriteLine("Added {0} known devices", sync.Devices.Count); } else { long addedVendors = 0; long addedProducts = 0; long addedOffsets = 0; long addedDevices = 0; long modifiedVendors = 0; long modifiedProducts = 0; long modifiedOffsets = 0; long modifiedDevices = 0; AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Updating USB vendors"); task.MaxValue = sync.UsbVendors.Count; foreach(UsbVendorDto vendor in sync.UsbVendors) { task.Increment(1); UsbVendor existing = mctx.UsbVendors.FirstOrDefault(v => v.Id == vendor.VendorId); if(existing != null) { modifiedVendors++; existing.Vendor = vendor.Vendor; existing.ModifiedWhen = updateStart; mctx.UsbVendors.Update(existing); } else { addedVendors++; mctx.UsbVendors.Add(new UsbVendor(vendor.VendorId, vendor.Vendor)); } } }); AaruConsole.WriteLine("Added {0} USB vendors", addedVendors); AaruConsole.WriteLine("Modified {0} USB vendors", modifiedVendors); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Updating USB products"); task.MaxValue = sync.UsbVendors.Count; foreach(UsbProductDto product in sync.UsbProducts) { task.Increment(1); UsbProduct existing = mctx.UsbProducts.FirstOrDefault(p => p.VendorId == product.VendorId && p.ProductId == product.ProductId); if(existing != null) { modifiedProducts++; existing.Product = product.Product; existing.ModifiedWhen = updateStart; mctx.UsbProducts.Update(existing); } else { addedProducts++; mctx.UsbProducts.Add(new UsbProduct(product.VendorId, product.ProductId, product.Product)); } } }); AaruConsole.WriteLine("Added {0} USB products", addedProducts); AaruConsole.WriteLine("Modified {0} USB products", modifiedProducts); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Updating CompactDisc read offsets"); task.MaxValue = sync.Offsets.Count; foreach(CdOffsetDto offset in sync.Offsets) { CdOffset existing = mctx.CdOffsets.FirstOrDefault(o => o.Id == offset.Id); task.Increment(1); if(existing != null) { modifiedOffsets++; existing.Agreement = offset.Agreement; existing.Manufacturer = offset.Manufacturer; existing.Model = offset.Model; existing.Submissions = offset.Submissions; existing.Offset = offset.Offset; existing.ModifiedWhen = updateStart; mctx.CdOffsets.Update(existing); } else { addedOffsets++; mctx.CdOffsets.Add(new CdOffset(offset) { Id = offset.Id }); } } }); AaruConsole.WriteLine("Added {0} CompactDisc read offsets", addedOffsets); AaruConsole.WriteLine("Modified {0} CompactDisc read offsets", modifiedOffsets); AnsiConsole.Progress().AutoClear(true).HideCompleted(true). Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()). Start(ctx => { ProgressTask task = ctx.AddTask("Updating known devices"); task.MaxValue = sync.Offsets.Count; foreach(DeviceDto device in sync.Devices) { task.Increment(1); Device existing = mctx.Devices.FirstOrDefault(d => d.Id == device.Id); if(existing != null) { modifiedDevices++; mctx.Remove(existing); existing = new Device(device) { Id = device.Id, OptimalMultipleSectorsRead = device.OptimalMultipleSectorsRead, CanReadGdRomUsingSwapDisc = device.CanReadGdRomUsingSwapDisc }; mctx.Devices.Add(existing); } else { addedDevices++; mctx.Devices.Add(new Device(device) { Id = device.Id, OptimalMultipleSectorsRead = device.OptimalMultipleSectorsRead, CanReadGdRomUsingSwapDisc = device.CanReadGdRomUsingSwapDisc }); } } }); AaruConsole.WriteLine("Added {0} known devices", addedDevices); AaruConsole.WriteLine("Modified {0} known devices", modifiedDevices); } } catch(Exception ex) { AaruConsole.ErrorWriteLine("Exception {0} when updating database.", ex); } finally { Spectre.ProgressSingleSpinner(ctx => { ctx.AddTask("Saving changes...").IsIndeterminate(); mctx.SaveChanges(); }); } } } }