diff --git a/Aaru.Server.Database/Models/BaseOperatingSystem.cs b/Aaru.Server.Database/Models/BaseOperatingSystem.cs index b3da167b..74e9d3b1 100644 --- a/Aaru.Server.Database/Models/BaseOperatingSystem.cs +++ b/Aaru.Server.Database/Models/BaseOperatingSystem.cs @@ -34,7 +34,7 @@ namespace Aaru.Server.Database.Models; public abstract class BaseOperatingSystem : BaseModel { - public string Name { get; set; } - public string Version { get; set; } - public long Count { get; set; } + public string Name { get; set; } = ""; + public string? Version { get; set; } + public long Count { get; set; } } \ No newline at end of file diff --git a/Aaru.Server.New/Controllers/UploadStatsController.cs b/Aaru.Server.New/Controllers/UploadStatsController.cs new file mode 100644 index 00000000..01148b6a --- /dev/null +++ b/Aaru.Server.New/Controllers/UploadStatsController.cs @@ -0,0 +1,393 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : UploadStatsController.cs +// Author(s) : Natalia Portillo +// +// Component : Aaru Server. +// +// --[ Description ] ---------------------------------------------------------- +// +// Handles statistics uploads. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2024 Natalia Portillo +// ****************************************************************************/ + +using System.Diagnostics; +using System.Net; +using System.Xml.Serialization; +using Aaru.CommonTypes.Metadata; +using Aaru.Server.Database.Models; +using Aaru.Server.New.Core; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; +using DbContext = Aaru.Server.Database.DbContext; +using OperatingSystem = Aaru.Server.Database.Models.OperatingSystem; +using Version = Aaru.Server.Database.Models.Version; + +namespace Aaru.Server.New.Controllers; + +public sealed class UploadStatsController : ControllerBase +{ + readonly DbContext _ctx; + + public UploadStatsController(DbContext ctx) => _ctx = ctx; + + /// Receives statistics from Aaru.Core, processes them and adds them to a server-side global statistics XML + /// HTTP response + [Route("api/uploadstats")] + [HttpPost] + public async Task UploadStats() + { + var response = new ContentResult + { + StatusCode = (int)HttpStatusCode.OK, + ContentType = "text/plain" + }; + + try + { + var newStats = new Stats(); + HttpRequest request = HttpContext.Request; + + var xs = new XmlSerializer(newStats.GetType()); + + newStats = (Stats)xs.Deserialize(new StringReader(await new StreamReader(request.Body).ReadToEndAsync())); + + if(newStats == null) + { + response.Content = "notstats"; + + return response; + } + + await StatsConverter.ConvertAsync(newStats); + + response.Content = "ok"; + + return response; + } + catch(Exception) + { +#if DEBUG + if(Debugger.IsAttached) throw; +#endif + response.Content = "error"; + + return response; + } + } + + /// Receives a report from Aaru.Core, verifies it's in the correct format and stores it on the server + /// HTTP response + [Route("api/uploadstatsv2")] + [HttpPost] + public async Task UploadStatsV2() + { + var response = new ContentResult + { + StatusCode = (int)HttpStatusCode.OK, + ContentType = "text/plain" + }; + + try + { + HttpRequest request = HttpContext.Request; + + var sr = new StreamReader(request.Body); + string statsString = await sr.ReadToEndAsync(); + StatsDto newstats = JsonConvert.DeserializeObject(statsString); + + if(newstats == null) + { + response.Content = "notstats"; + + return response; + } + + if(newstats.Commands != null) + { + foreach(NameValueStats nvs in newstats.Commands) + { + if(nvs.name == "analyze") nvs.name = "fs-info"; + + Command? existing = await _ctx.Commands.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.Commands.Add(new Command + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.Versions != null) + { + foreach(NameValueStats nvs in newstats.Versions) + { + Version? existing = await _ctx.Versions.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.Versions.Add(new Version + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.Filesystems != null) + { + foreach(NameValueStats nvs in newstats.Filesystems) + { + Filesystem? existing = await _ctx.Filesystems.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.Filesystems.Add(new Filesystem + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.Partitions != null) + { + foreach(NameValueStats nvs in newstats.Partitions) + { + Partition? existing = await _ctx.Partitions.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.Partitions.Add(new Partition + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.MediaFormats != null) + { + foreach(NameValueStats nvs in newstats.MediaFormats) + { + MediaFormat? existing = await _ctx.MediaFormats.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.MediaFormats.Add(new MediaFormat + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.Filters != null) + { + foreach(NameValueStats nvs in newstats.Filters) + { + Filter? existing = await _ctx.Filters.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.Filters.Add(new Filter + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.OperatingSystems != null) + { + foreach(OsStats operatingSystem in newstats.OperatingSystems) + { + OperatingSystem? existing = + await _ctx.OperatingSystems.FirstOrDefaultAsync(c => c.Name == operatingSystem.name && + c.Version == operatingSystem.version); + + if(existing == null) + { + _ctx.OperatingSystems.Add(new OperatingSystem + { + Name = operatingSystem.name, + Version = operatingSystem.version, + Count = operatingSystem.Value + }); + } + else + existing.Count += operatingSystem.Value; + } + } + + if(newstats.Medias != null) + { + foreach(MediaStats media in newstats.Medias) + { + Media? existing = + await _ctx.Medias.FirstOrDefaultAsync(c => c.Type == media.type && c.Real == media.real); + + if(existing == null) + { + _ctx.Medias.Add(new Media + { + Type = media.type, + Real = media.real, + Count = media.Value + }); + } + else + existing.Count += media.Value; + } + } + + if(newstats.Devices != null) + { + foreach(DeviceStats device in newstats.Devices) + { + DeviceStat? existing = + await _ctx.DeviceStats.FirstOrDefaultAsync(c => c.Bus == device.Bus && + c.Manufacturer == device.Manufacturer && + c.Model == device.Model && + c.Revision == device.Revision); + + if(existing == null) + { + _ctx.DeviceStats.Add(new DeviceStat + { + Bus = device.Bus, + Manufacturer = device.Manufacturer, + Model = device.Model, + Revision = device.Revision + }); + } + } + } + + if(newstats.RemoteApplications != null) + { + foreach(OsStats application in newstats.RemoteApplications) + { + RemoteApplication? existing = + await _ctx.RemoteApplications.FirstOrDefaultAsync(c => c.Name == application.name && + c.Version == application.version); + + if(existing == null) + { + _ctx.RemoteApplications.Add(new RemoteApplication + { + Name = application.name, + Version = application.version, + Count = application.Value + }); + } + else + existing.Count += application.Value; + } + } + + if(newstats.RemoteArchitectures != null) + { + foreach(NameValueStats nvs in newstats.RemoteArchitectures) + { + RemoteArchitecture? existing = + await _ctx.RemoteArchitectures.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + _ctx.RemoteArchitectures.Add(new RemoteArchitecture + { + Name = nvs.name, + Count = nvs.Value + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newstats.RemoteOperatingSystems != null) + { + foreach(OsStats remoteOperatingSystem in newstats.RemoteOperatingSystems) + { + RemoteOperatingSystem? existing = + await _ctx.RemoteOperatingSystems.FirstOrDefaultAsync(c => + c.Name == + remoteOperatingSystem.name && + c.Version == + remoteOperatingSystem.version); + + if(existing == null) + { + _ctx.RemoteOperatingSystems.Add(new RemoteOperatingSystem + { + Name = remoteOperatingSystem.name, + Version = remoteOperatingSystem.version, + Count = remoteOperatingSystem.Value + }); + } + else + existing.Count += remoteOperatingSystem.Value; + } + } + + await _ctx.SaveChangesAsync(); + + response.Content = "ok"; + + return response; + } + + // ReSharper disable once RedundantCatchClause + catch + { +#if DEBUG + if(Debugger.IsAttached) throw; +#endif + response.Content = "error"; + + return response; + } + } +} \ No newline at end of file diff --git a/Aaru.Server.New/Core/StatsConverter.cs b/Aaru.Server.New/Core/StatsConverter.cs new file mode 100644 index 00000000..640fa39d --- /dev/null +++ b/Aaru.Server.New/Core/StatsConverter.cs @@ -0,0 +1,590 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : StatsConverter.cs +// Author(s) : Natalia Portillo +// +// Component : Aaru Server. +// +// --[ Description ] ---------------------------------------------------------- +// +// Reads a statistics XML and stores it in the database context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2024 Natalia Portillo +// ****************************************************************************/ + +using Aaru.CommonTypes.Metadata; +using Aaru.Server.Database.Models; +using Microsoft.EntityFrameworkCore; +using DbContext = Aaru.Server.Database.DbContext; +using OperatingSystem = Aaru.Server.Database.Models.OperatingSystem; +using Version = Aaru.Server.Database.Models.Version; + +namespace Aaru.Server.New.Core; + +public static class StatsConverter +{ + public static async Task ConvertAsync(Stats newStats) + { + var ctx = new DbContext(); + + if(newStats.Commands?.Analyze > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "fs-info"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Analyze, + Name = "fs-info" + }); + } + else + existing.Count += newStats.Commands.Analyze; + } + + if(newStats.Commands?.Benchmark > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "benchmark"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Benchmark, + Name = "benchmark" + }); + } + else + existing.Count += newStats.Commands.Benchmark; + } + + if(newStats.Commands?.Checksum > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "checksum"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Checksum, + Name = "checksum" + }); + } + else + existing.Count += newStats.Commands.Checksum; + } + + if(newStats.Commands?.Compare > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "compare"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Compare, + Name = "compare" + }); + } + else + existing.Count += newStats.Commands.Compare; + } + + if(newStats.Commands?.CreateSidecar > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "create-sidecar"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.CreateSidecar, + Name = "create-sidecar" + }); + } + else + existing.Count += newStats.Commands.CreateSidecar; + } + + if(newStats.Commands?.Decode > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "decode"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Decode, + Name = "decode" + }); + } + else + existing.Count += newStats.Commands.Decode; + } + + if(newStats.Commands?.DeviceInfo > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "device-info"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.DeviceInfo, + Name = "device-info" + }); + } + else + existing.Count += newStats.Commands.DeviceInfo; + } + + if(newStats.Commands?.DeviceReport > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "device-report"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.DeviceReport, + Name = "device-report" + }); + } + else + existing.Count += newStats.Commands.DeviceReport; + } + + if(newStats.Commands?.DumpMedia > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "dump-media"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.DumpMedia, + Name = "dump-media" + }); + } + else + existing.Count += newStats.Commands.DumpMedia; + } + + if(newStats.Commands?.Entropy > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "entropy"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Entropy, + Name = "entropy" + }); + } + else + existing.Count += newStats.Commands.Entropy; + } + + if(newStats.Commands?.Formats > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "formats"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Formats, + Name = "formats" + }); + } + else + existing.Count += newStats.Commands.Formats; + } + + if(newStats.Commands?.MediaInfo > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "media-info"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.MediaInfo, + Name = "media-info" + }); + } + else + existing.Count += newStats.Commands.MediaInfo; + } + + if(newStats.Commands?.MediaScan > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "media-scan"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.MediaScan, + Name = "media-scan" + }); + } + else + existing.Count += newStats.Commands.MediaScan; + } + + if(newStats.Commands?.PrintHex > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "printhex"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.PrintHex, + Name = "printhex" + }); + } + else + existing.Count += newStats.Commands.PrintHex; + } + + if(newStats.Commands?.Verify > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "verify"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Verify, + Name = "verify" + }); + } + else + existing.Count += newStats.Commands.Verify; + } + + if(newStats.Commands?.Ls > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "ls"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.Ls, + Name = "ls" + }); + } + else + existing.Count += newStats.Commands.Ls; + } + + if(newStats.Commands?.ExtractFiles > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "extract-files"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.ExtractFiles, + Name = "extract-files" + }); + } + else + existing.Count += newStats.Commands.ExtractFiles; + } + + if(newStats.Commands?.ListDevices > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "list-devices"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.ListDevices, + Name = "list-devices" + }); + } + else + existing.Count += newStats.Commands.ListDevices; + } + + if(newStats.Commands?.ListEncodings > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "list-encodings"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.ListEncodings, + Name = "list-encodings" + }); + } + else + existing.Count += newStats.Commands.ListEncodings; + } + + if(newStats.Commands?.ConvertImage > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "convert-image"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.ConvertImage, + Name = "convert-image" + }); + } + else + existing.Count += newStats.Commands.ConvertImage; + } + + if(newStats.Commands?.ImageInfo > 0) + { + Command? existing = await ctx.Commands.FirstOrDefaultAsync(static c => c.Name == "image-info"); + + if(existing == null) + { + ctx.Commands.Add(new Command + { + Count = newStats.Commands.ImageInfo, + Name = "image-info" + }); + } + else + existing.Count += newStats.Commands.ImageInfo; + } + + if(newStats.OperatingSystems != null) + { + foreach(OsStats operatingSystem in newStats.OperatingSystems) + { + if(string.IsNullOrWhiteSpace(operatingSystem.name) || + string.IsNullOrWhiteSpace(operatingSystem.version)) + continue; + + OperatingSystem? existing = + await ctx.OperatingSystems.FirstOrDefaultAsync(c => c.Name == operatingSystem.name && + c.Version == operatingSystem.version); + + if(existing == null) + { + ctx.OperatingSystems.Add(new OperatingSystem + { + Count = operatingSystem.Value, + Name = operatingSystem.name, + Version = operatingSystem.version + }); + } + else + existing.Count += operatingSystem.Value; + } + } + else + { + OperatingSystem? existing = + await ctx.OperatingSystems.FirstOrDefaultAsync(static c => c.Name == "Linux" && c.Version == null); + + if(existing == null) + { + ctx.OperatingSystems.Add(new OperatingSystem + { + Count = 1, + Name = "Linux" + }); + } + else + existing.Count++; + } + + if(newStats.Versions != null) + { + foreach(NameValueStats nvs in newStats.Versions) + { + if(string.IsNullOrWhiteSpace(nvs.name)) continue; + + Version? existing = await ctx.Versions.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + ctx.Versions.Add(new Version + { + Count = nvs.Value, + Name = nvs.name + }); + } + else + existing.Count += nvs.Value; + } + } + else + { + Version? existing = await ctx.Versions.FirstOrDefaultAsync(static c => c.Name == "previous"); + + if(existing == null) + { + ctx.Versions.Add(new Version + { + Count = 1, + Name = "previous" + }); + } + else + existing.Count++; + } + + if(newStats.Filesystems != null) + { + foreach(NameValueStats nvs in newStats.Filesystems) + { + if(string.IsNullOrWhiteSpace(nvs.name)) continue; + + Filesystem? existing = await ctx.Filesystems.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + ctx.Filesystems.Add(new Filesystem + { + Count = nvs.Value, + Name = nvs.name + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newStats.Partitions != null) + { + foreach(NameValueStats nvs in newStats.Partitions) + { + if(string.IsNullOrWhiteSpace(nvs.name)) continue; + + Partition? existing = await ctx.Partitions.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + ctx.Partitions.Add(new Partition + { + Count = nvs.Value, + Name = nvs.name + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newStats.MediaImages != null) + { + foreach(NameValueStats nvs in newStats.MediaImages) + { + if(string.IsNullOrWhiteSpace(nvs.name)) continue; + + MediaFormat? existing = await ctx.MediaFormats.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + ctx.MediaFormats.Add(new MediaFormat + { + Count = nvs.Value, + Name = nvs.name + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newStats.Filters != null) + { + foreach(NameValueStats nvs in newStats.Filters) + { + if(string.IsNullOrWhiteSpace(nvs.name)) continue; + + Filter? existing = await ctx.Filters.FirstOrDefaultAsync(c => c.Name == nvs.name); + + if(existing == null) + { + ctx.Filters.Add(new Filter + { + Count = nvs.Value, + Name = nvs.name + }); + } + else + existing.Count += nvs.Value; + } + } + + if(newStats.Devices != null) + { + foreach(DeviceStats device in newStats.Devices + .Where(static device => !string.IsNullOrWhiteSpace(device.Model)) + .Where(device => !ctx.DeviceStats.Any(c => c.Bus == device.Bus && + c.Manufacturer == device.Manufacturer && + c.Model == device.Model && + c.Revision == device.Revision))) + { + ctx.DeviceStats.Add(new DeviceStat + { + Bus = device.Bus, + Manufacturer = device.Manufacturer, + Model = device.Model, + Revision = device.Revision + }); + } + } + + if(newStats.Medias != null) + { + foreach(MediaStats media in newStats.Medias) + { + if(string.IsNullOrWhiteSpace(media.type)) continue; + + Media? existing = + await ctx.Medias.FirstOrDefaultAsync(c => c.Type == media.type && c.Real == media.real); + + if(existing == null) + { + ctx.Medias.Add(new Media + { + Count = media.Value, + Real = media.real, + Type = media.type + }); + } + else + existing.Count += media.Value; + } + } + + await ctx.SaveChangesAsync(); + } +} \ No newline at end of file