Aaru/Aaru.Core/Statistics.cs

// /***************************************************************************
// Aaru Data Preservation Suite
// ----------------------------------------------------------------------------
//
// Filename       : Statistics.cs
// Author(s)      : Natalia Portillo <claunia@claunia.com>
//
// Component      : Core algorithms.
//
// --[ Description ] ----------------------------------------------------------
//
//     Handles usage statistics.
//
// --[ License ] --------------------------------------------------------------
//
//     This program is free software: you can redistribute it and/or modify
//     it under the terms of the GNU General Public License as
//     published by the Free Software Foundation, either version 3 of the
//     License, or (at your option) any later version.
//
//     This program is distributed in the hope that it will be useful,
//     but WITHOUT ANY WARRANTY; without even the implied warranty of
//     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//     GNU General Public License for more details.
//
//     You should have received a copy of the GNU General Public License
//     along with this program.  If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2025 Natalia Portillo
// ****************************************************************************/

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using Aaru.CommonTypes.Interop;
using Aaru.CommonTypes.Metadata;
using Aaru.Database;
using Aaru.Database.Models;
using Aaru.Logging;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Sentry;
using Device = Aaru.Devices.Device;
using MediaType = Aaru.CommonTypes.MediaType;
using OperatingSystem = Aaru.Database.Models.OperatingSystem;
using Version = Aaru.Database.Models.Version;

namespace Aaru.Core;

/// <summary>Handles anonymous usage statistics</summary>
public static class Statistics
{
    const string MODULE_NAME = "Stats";
    /// <summary>Statistics file semaphore</summary>
    static bool _submitStatsLock;

    /// <summary>Loads saved statistics from disk</summary>
    public static void LoadStats()
    {
        try
        {
            using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

            if(Settings.Settings.Current.Stats == null) return;

            ctx.OperatingSystems.Add(new OperatingSystem
            {
                Name         = DetectOS.GetRealPlatformID().ToString(),
                Synchronized = false,
                Version      = DetectOS.GetVersion(),
                Count        = 1
            });

            ctx.Versions.Add(new Version
            {
                Name         = CommonTypes.Interop.Version.GetInformationalVersion(),
                Synchronized = false,
                Count        = 1
            });

            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Saves statistics to disk</summary>
    public static async Task SaveStatsAsync()
    {
        try
        {
            await using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

            await ctx.SaveChangesAsync();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }

        if(Settings.Settings.Current.Stats is { ShareStats: true }) await SubmitStatsAsync();
    }

    /// <summary>Submits statistics to Aaru.Server</summary>
    static async Task SubmitStatsAsync()
    {
        await using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        try
        {
            if(_submitStatsLock) return;

            _submitStatsLock = true;
            var dto = new StatsDto();

            AddStats(ctx.Commands, out List<NameValueStats> nameValueStats);

            if(nameValueStats?.Count > 0) dto.Commands = nameValueStats;

            AddStats(ctx.Filesystems, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.Filesystems = nameValueStats;

            AddStats(ctx.Filters, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.Filters = nameValueStats;

            AddStats(ctx.MediaFormats, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.MediaFormats = nameValueStats;

            AddStats(ctx.Archives, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.Archives = nameValueStats;

            AddStats(ctx.Partitions, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.Partitions = nameValueStats;

            AddStats(ctx.Versions, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.Versions = nameValueStats;

            if(ctx.Medias.Any(static c => !c.Synchronized))
            {
                dto.Medias = [];

                foreach(string media in ctx.Medias.Where(static c => !c.Synchronized)
                                           .Select(static c => c.Type)
                                           .Distinct())
                {
                    if(ctx.Medias.Any(c => !c.Synchronized && c.Type == media && c.Real))
                    {
                        dto.Medias.Add(new MediaStats
                        {
                            real      = true,
                            MediaType = media,
                            Value     = ctx.Medias.LongCount(c => !c.Synchronized && c.Type == media && c.Real)
                        });
                    }

                    if(ctx.Medias.Any(c => !c.Synchronized && c.Type == media && !c.Real))
                    {
                        dto.Medias.Add(new MediaStats
                        {
                            real      = false,
                            MediaType = media,
                            Value     = ctx.Medias.LongCount(c => !c.Synchronized && c.Type == media && !c.Real)
                        });
                    }
                }
            }

            if(ctx.SeenDevices.Any(static c => !c.Synchronized))
            {
                dto.Devices = [];

                foreach(DeviceStat device in ctx.SeenDevices.Where(static c => !c.Synchronized))
                {
                    dto.Devices.Add(new DeviceStats
                    {
                        Bus                   = device.Bus,
                        Manufacturer          = device.Manufacturer,
                        ManufacturerSpecified = device.Manufacturer is not null,
                        Model                 = device.Model,
                        Revision              = device.Revision
                    });
                }
            }

            AddOperatingSystem(ctx.OperatingSystems, out List<OsStats> osStats);
            if(nameValueStats?.Count > 0) dto.OperatingSystems = osStats;

            AddOperatingSystem(ctx.RemoteApplications, out osStats);
            if(nameValueStats?.Count > 0) dto.RemoteApplications = osStats;

            AddStats(ctx.RemoteArchitectures, out nameValueStats);

            if(nameValueStats?.Count > 0) dto.RemoteArchitectures = nameValueStats;

            AddOperatingSystem(ctx.RemoteOperatingSystems, out osStats);
            if(nameValueStats?.Count > 0) dto.RemoteOperatingSystems = osStats;

#if DEBUG
            Console.WriteLine(Localization.Core.Uploading_statistics);
#else
            Aaru.Logging.AaruLogging.Debug(MODULE_NAME, Localization.Core.Uploading_statistics);
#endif
            using StringContent jsonContent =
                new(JsonSerializer.Serialize(dto, typeof(StatsDto), StatsDtoContext.Default),
                    Encoding.UTF8,
                    "application/json");

            var client = new HttpClient();
            client.BaseAddress = new Uri("https://www.aaru.app");
            client.DefaultRequestHeaders.Add("User-Agent", $"Aaru {typeof(Version).Assembly.GetName().Version}");

            using HttpResponseMessage response = await client.PostAsync("/api/uploadstatsv2", jsonContent);

            if(response.StatusCode != HttpStatusCode.OK) return;

            string result = await response.Content.ReadAsStringAsync();

            if(result != "ok") return;

            await UpdateStatsAsync(ctx.Commands);
            await UpdateStatsAsync(ctx.Filesystems);
            await UpdateStatsAsync(ctx.Filters);
            await UpdateStatsAsync(ctx.MediaFormats);
            await UpdateStatsAsync(ctx.Archives);
            await UpdateStatsAsync(ctx.Partitions);
            await UpdateStatsAsync(ctx.Versions);

            if(ctx.Medias.Any(static c => !c.Synchronized))
            {
                foreach(string media in ctx.Medias.Where(static c => !c.Synchronized)
                                           .Select(static c => c.Type)
                                           .Distinct())
                {
                    if(ctx.Medias.Any(c => !c.Synchronized && c.Type == media && c.Real))
                    {
                        Database.Models.Media existing =
                            await ctx.Medias.FirstOrDefaultAsync(c => c.Synchronized && c.Type == media && c.Real) ??
                            new Database.Models.Media
                            {
                                Synchronized = true,
                                Type         = media,
                                Real         = true
                            };

                        existing.Count +=
                            (ulong)ctx.Medias.LongCount(c => !c.Synchronized && c.Type == media && c.Real);

                        ctx.Medias.Update(existing);

                        ctx.Medias.RemoveRange(ctx.Medias.Where(c => !c.Synchronized && c.Type == media && c.Real));
                    }

                    if(!ctx.Medias.Any(c => !c.Synchronized && c.Type == media && !c.Real)) continue;

                    {
                        Database.Models.Media existing =
                            await ctx.Medias.FirstOrDefaultAsync(c => c.Synchronized && c.Type == media && !c.Real) ??
                            new Database.Models.Media
                            {
                                Synchronized = true,
                                Type         = media,
                                Real         = false
                            };

                        existing.Count +=
                            (ulong)ctx.Medias.LongCount(c => !c.Synchronized && c.Type == media && !c.Real);

                        ctx.Medias.Update(existing);

                        ctx.Medias.RemoveRange(ctx.Medias.Where(c => !c.Synchronized && c.Type == media && !c.Real));
                    }
                }
            }

            if(ctx.SeenDevices.Any(static c => !c.Synchronized))
            {
                foreach(DeviceStat device in ctx.SeenDevices.Where(static c => !c.Synchronized))
                {
                    device.Synchronized = true;
                    ctx.Update(device);
                }
            }

            await UpdateOperatingSystemAsync(ctx.OperatingSystems);
            await UpdateOperatingSystemAsync(ctx.RemoteApplications);
            await UpdateStatsAsync(ctx.RemoteArchitectures);
            await UpdateOperatingSystemAsync(ctx.RemoteOperatingSystems);

            await ctx.SaveChangesAsync();
        }
        catch(WebException ex)
        {
            // Can't connect to the server, do nothing
            SentrySdk.CaptureException(ex);
        }
        catch(DbUpdateConcurrencyException ex)
        {
            // Ignore db concurrency errors
            SentrySdk.CaptureException(ex);
        }

        catch(Exception ex)
        {
            SentrySdk.CaptureException(ex);

#if DEBUG
            _submitStatsLock = false;

            if(Debugger.IsAttached) throw;
#endif
        }

        _submitStatsLock = false;
    }

    static async Task UpdateOperatingSystemAsync<T>(DbSet<T> source) where T : BaseOperatingSystem, new()
    {
        if(!source.Any(static c => !c.Synchronized)) return;

        foreach(string name in source.Where(static c => !c.Synchronized).Select(static c => c.Name).Distinct())
        {
            foreach(string version in source.Where(c => !c.Synchronized && c.Name == name)
                                            .Select(static c => c.Version)
                                            .Distinct())
            {
                T existing =
                    await source.FirstOrDefaultAsync(c => c.Synchronized && c.Name == name && c.Version == version) ??
                    new T
                    {
                        Synchronized = true,
                        Version      = version,
                        Name         = name
                    };

                existing.Count +=
                    (ulong)source.LongCount(c => !c.Synchronized && c.Name == name && c.Version == version);

                source.Update(existing);

                source.RemoveRange(source.Where(c => !c.Synchronized && c.Name == name && c.Version == version));
            }
        }
    }

    static async Task UpdateStatsAsync<T>(DbSet<T> source) where T : NameCountModel, new()
    {
        if(!source.Any(static c => !c.Synchronized)) return;

        foreach(string nvs in source.Where(static c => !c.Synchronized).Select(static c => c.Name).Distinct())
        {
            T existing = await source.FirstOrDefaultAsync(c => c.Synchronized && c.Name == nvs) ??
                         new T
                         {
                             Name         = nvs,
                             Synchronized = true
                         };

            existing.Count += (ulong)source.LongCount(c => !c.Synchronized && c.Name == nvs);
            source.Update(existing);
            source.RemoveRange(source.Where(c => !c.Synchronized && c.Name == nvs));
        }
    }

    static void AddOperatingSystem(IQueryable<BaseOperatingSystem> source, out List<OsStats> destination)
    {
        destination = [];

        foreach(string remoteOsName in source.Where(static c => !c.Synchronized).Select(static c => c.Name).Distinct())
        {
            foreach(string remoteOsVersion in source.Where(c => !c.Synchronized && c.Name == remoteOsName)
                                                    .Select(static c => c.Version)
                                                    .Distinct())
            {
                destination.Add(new OsStats
                {
                    name    = remoteOsName,
                    version = remoteOsVersion,
                    Value = source.LongCount(c => !c.Synchronized           &&
                                                  c.Name    == remoteOsName &&
                                                  c.Version == remoteOsVersion)
                });
            }
        }
    }

    static void AddStats(IQueryable<NameCountModel> source, out List<NameValueStats> destination)
    {
        destination = [];

        if(!source.Any(static c => !c.Synchronized)) return;

        foreach(string nvs in source.Where(static c => !c.Synchronized).Select(static c => c.Name).Distinct())
        {
            destination.Add(new NameValueStats
            {
                name  = nvs,
                Value = source.LongCount(c => !c.Synchronized && c.Name == nvs)
            });
        }
    }

    /// <summary>Adds the execution of a command to statistics</summary>
    /// <param name="command">Command</param>
    public static void AddCommand(string command)
    {
        if(string.IsNullOrWhiteSpace(command)) return;

        if(Settings.Settings.Current.Stats is not { DeviceStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Commands.Add(new Command
        {
            Name         = command,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new filesystem to statistics</summary>
    /// <param name="filesystem">Filesystem name</param>
    public static void AddFilesystem(string filesystem)
    {
        if(string.IsNullOrWhiteSpace(filesystem)) return;

        if(Settings.Settings.Current.Stats is not { FilesystemStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Filesystems.Add(new Filesystem
        {
            Name         = filesystem,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new partition scheme to statistics</summary>
    /// <param name="partition">Partition scheme name</param>
    internal static void AddPartition(string partition)
    {
        if(string.IsNullOrWhiteSpace(partition)) return;

        if(Settings.Settings.Current.Stats is not { PartitionStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Partitions.Add(new Partition
        {
            Name         = partition,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new filter to statistics</summary>
    /// <param name="filter">Filter name</param>
    public static void AddFilter(string filter)
    {
        if(string.IsNullOrWhiteSpace(filter)) return;

        if(Settings.Settings.Current.Stats is not { FilterStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Filters.Add(new Filter
        {
            Name         = filter,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Ads a new media image to statistics</summary>
    /// <param name="format">Media image name</param>
    public static void AddMediaFormat(string format)
    {
        if(string.IsNullOrWhiteSpace(format)) return;

        if(Settings.Settings.Current.Stats is not { MediaImageStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.MediaFormats.Add(new MediaFormat
        {
            Name         = format,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new device to statistics</summary>
    /// <param name="dev">Device</param>
    public static void AddDevice(Device dev)
    {
        if(Settings.Settings.Current.Stats is not { DeviceStats: true }) return;

        string deviceBus;

        if(dev.IsUsb)
            deviceBus = "USB";
        else if(dev.IsFireWire)
            deviceBus = "FireWire";
        else
            deviceBus = dev.Type.ToString();

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        if(ctx.SeenDevices.Any(d => d.Manufacturer == dev.Manufacturer     &&
                                    d.Model        == dev.Model            &&
                                    d.Revision     == dev.FirmwareRevision &&
                                    d.Bus          == deviceBus))
            return;

        ctx.SeenDevices.Add(new DeviceStat
        {
            Bus          = deviceBus,
            Manufacturer = dev.Manufacturer,
            Model        = dev.Model,
            Revision     = dev.FirmwareRevision,
            Synchronized = false
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new media type to statistics</summary>
    /// <param name="type">Media type</param>
    /// <param name="real">Set if media was found on a real device, otherwise found on a media image</param>
    public static void AddMedia(MediaType type, bool real)
    {
        if(Settings.Settings.Current.Stats is not { MediaStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Medias.Add(new Database.Models.Media
        {
            Real         = real,
            Synchronized = false,
            Type         = type.ToString(),
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Adds a new remote to statistics</summary>
    public static void AddRemote(string serverApplication, string serverVersion, string serverOperatingSystem,
                                 string serverOperatingSystemVersion, string serverArchitecture)
    {
        if(Settings.Settings.Current.Stats is not { MediaStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.RemoteApplications.Add(new RemoteApplication
        {
            Count        = 1,
            Name         = serverApplication,
            Synchronized = false,
            Version      = serverVersion
        });

        ctx.RemoteArchitectures.Add(new RemoteArchitecture
        {
            Count        = 1,
            Name         = serverArchitecture,
            Synchronized = false
        });

        ctx.RemoteOperatingSystems.Add(new RemoteOperatingSystem
        {
            Count        = 1,
            Name         = serverOperatingSystem,
            Synchronized = false,
            Version      = serverOperatingSystemVersion
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }

    /// <summary>Ads a new archive to statistics</summary>
    /// <param name="format">Archive format name</param>
    public static void AddArchiveFormat(string format)
    {
        if(string.IsNullOrWhiteSpace(format)) return;

        if(Settings.Settings.Current.Stats is not { MediaStats: true }) return;

        using var ctx = AaruContext.Create(Settings.Settings.LocalDbPath);

        ctx.Archives.Add(new Archive
        {
            Name         = format,
            Synchronized = false,
            Count        = 1
        });

        try
        {
            ctx.SaveChanges();
        }
        catch(SqliteException ex)
        {
            AaruLogging.Debug(MODULE_NAME, Localization.Core.Exception_while_trying_to_save_statistics);
            AaruLogging.Exception(ex, Localization.Core.Exception_while_trying_to_save_statistics);
        }
    }
}