From a219d0c5de2c59bf9efdb4078eca21aaa9968a5f Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Tue, 16 Jul 2024 14:07:18 -0400 Subject: [PATCH] Add some client helper classes --- SabreTools.RedumpLib/Web/DelayHelper.cs | 21 +++++ SabreTools.RedumpLib/Web/Discs.cs | 55 +++++++++++++ SabreTools.RedumpLib/Web/Packs.cs | 73 +++++++++++++++++ SabreTools.RedumpLib/Web/Search.cs | 93 ++++++++++++++++++++++ SabreTools.RedumpLib/Web/User.cs | 100 ++++++++++++++++++++++++ SabreTools.RedumpLib/Web/WIP.cs | 46 +++++++++++ 6 files changed, 388 insertions(+) create mode 100644 SabreTools.RedumpLib/Web/DelayHelper.cs create mode 100644 SabreTools.RedumpLib/Web/Discs.cs create mode 100644 SabreTools.RedumpLib/Web/Packs.cs create mode 100644 SabreTools.RedumpLib/Web/Search.cs create mode 100644 SabreTools.RedumpLib/Web/User.cs create mode 100644 SabreTools.RedumpLib/Web/WIP.cs diff --git a/SabreTools.RedumpLib/Web/DelayHelper.cs b/SabreTools.RedumpLib/Web/DelayHelper.cs new file mode 100644 index 0000000..2c33790 --- /dev/null +++ b/SabreTools.RedumpLib/Web/DelayHelper.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Helper class for delaying + /// + internal static class DelayHelper + { + /// + /// Delay a random amount of time up to 5 seconds + /// + public static void DelayRandom() + { + var r = new Random(); + int delay = r.Next(0, 50); + Thread.Sleep(delay * 100); + } + } +} \ No newline at end of file diff --git a/SabreTools.RedumpLib/Web/Discs.cs b/SabreTools.RedumpLib/Web/Discs.cs new file mode 100644 index 0000000..d576b13 --- /dev/null +++ b/SabreTools.RedumpLib/Web/Discs.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading.Tasks; +using SabreTools.RedumpLib.Data; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Contains logic for dealing with disc pages + /// + public static class Discs + { + /// + /// Download the last modified disc pages, until first failure + /// + /// RedumpClient for connectivity + /// Output directory to save data to + /// Force continuation of download + public static async Task DownloadLastModified(RedumpClient rc, string? outDir, bool force) + { + // Keep getting last modified pages until there are none left + int pageNumber = 1; + while (true) + { + if (!await rc.CheckSingleSitePage(string.Format(Constants.LastModifiedUrl, pageNumber++), outDir, !force)) + break; + } + + return true; + } + + /// + /// Download the specified range of site disc pages + /// + /// RedumpClient for connectivity + /// Output directory to save data to + /// Starting ID for the range + /// Ending ID for the range (inclusive) + public static async Task DownloadSiteRange(RedumpClient rc, string? outDir, int minId = 0, int maxId = 0) + { + if (!rc.LoggedIn) + { + Console.WriteLine("Site download functionality is only available to Redump members"); + return false; + } + + for (int id = minId; id <= maxId; id++) + { + if (await rc.DownloadSingleSiteID(id, outDir, true)) + DelayHelper.DelayRandom(); // Intentional sleep here so we don't flood the server + } + + return true; + } + } +} \ No newline at end of file diff --git a/SabreTools.RedumpLib/Web/Packs.cs b/SabreTools.RedumpLib/Web/Packs.cs new file mode 100644 index 0000000..03b4c27 --- /dev/null +++ b/SabreTools.RedumpLib/Web/Packs.cs @@ -0,0 +1,73 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using SabreTools.RedumpLib.Data; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Contains logic for dealing with packs + /// + internal static class Packs + { + /// + /// Download premade packs + /// + /// RedumpClient for connectivity + /// Output directory to save data to + /// True to use named subfolders to store downloads, false to store directly in the output directory + public static async Task DownloadPacks(RedumpClient rc, string? outDir, bool useSubfolders) + { +#if NETFRAMEWORK || NETCOREAPP3_1 + var systems = Enum.GetValues(typeof(RedumpSystem)).OfType().Select(s => new Nullable(s)); +#else + var systems = Enum.GetValues().Select(s => new RedumpSystem?(s)); +#endif + + await rc.DownloadPacks(Constants.PackCuesUrl, systems.Where(s => s.HasCues()).ToArray(), "CUEs", outDir, useSubfolders ? "cue" : null); + await rc.DownloadPacks(Constants.PackDatfileUrl, systems.Where(s => s.HasDat()).ToArray(), "DATs", outDir, useSubfolders ? "dat" : null); + await rc.DownloadPacks(Constants.PackDkeysUrl, systems.Where(s => s.HasDkeys()).ToArray(), "Decrypted KEYS", outDir, useSubfolders ? "dkey" : null); + await rc.DownloadPacks(Constants.PackGdiUrl, systems.Where(s => s.HasGdi()).ToArray(), "GDIs", outDir, useSubfolders ? "gdi" : null); + await rc.DownloadPacks(Constants.PackKeysUrl, systems.Where(s => s.HasKeys()).ToArray(), "KEYS", outDir, useSubfolders ? "keys" : null); + await rc.DownloadPacks(Constants.PackLsdUrl, systems.Where(s => s.HasLsd()).ToArray(), "LSD", outDir, useSubfolders ? "lsd" : null); + await rc.DownloadPacks(Constants.PackSbiUrl, systems.Where(s => s.HasSbi()).ToArray(), "SBIs", outDir, useSubfolders ? "sbi" : null); + + return true; + } + + /// + /// Download premade packs for an individual system + /// + /// RedumpClient for connectivity + /// RedumpSystem to get all possible packs for + /// Output directory to save data to + /// True to use named subfolders to store downloads, false to store directly in the output directory + public static async Task DownloadPacksForSystem(RedumpClient rc, RedumpSystem? system, string? outDir, bool useSubfolders) + { + var systemAsArray = new RedumpSystem?[] { system }; + + if (system.HasCues()) + await rc.DownloadPacks(Constants.PackCuesUrl, systemAsArray, "CUEs", outDir, useSubfolders ? "cue" : null); + + if (system.HasDat()) + await rc.DownloadPacks(Constants.PackDatfileUrl, systemAsArray, "DATs", outDir, useSubfolders ? "dat" : null); + + if (system.HasDkeys()) + await rc.DownloadPacks(Constants.PackDkeysUrl, systemAsArray, "Decrypted KEYS", outDir, useSubfolders ? "dkey" : null); + + if (system.HasGdi()) + await rc.DownloadPacks(Constants.PackGdiUrl, systemAsArray, "GDIs", outDir, useSubfolders ? "gdi" : null); + + if (system.HasKeys()) + await rc.DownloadPacks(Constants.PackKeysUrl, systemAsArray, "KEYS", outDir, useSubfolders ? "keys" : null); + + if (system.HasLsd()) + await rc.DownloadPacks(Constants.PackLsdUrl, systemAsArray, "LSD", outDir, useSubfolders ? "lsd" : null); + + if (system.HasSbi()) + await rc.DownloadPacks(Constants.PackSbiUrl, systemAsArray, "SBIs", outDir, useSubfolders ? "sbi" : null); + + return true; + } + } +} \ No newline at end of file diff --git a/SabreTools.RedumpLib/Web/Search.cs b/SabreTools.RedumpLib/Web/Search.cs new file mode 100644 index 0000000..ed7b99c --- /dev/null +++ b/SabreTools.RedumpLib/Web/Search.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using SabreTools.RedumpLib.Data; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Contains logic for dealing with searches + /// + internal static class Search + { + /// + /// List the disc IDs associated with a given quicksearch query + /// + /// RedumpClient for connectivity + /// Query string to attempt to search for + /// All disc IDs for the given query, null on error + public static async Task?> ListSearchResults(RedumpClient rc, string? query) + { + // If the query is invalid + if (string.IsNullOrEmpty(query)) + return null; + + List ids = []; + + // Strip quotes + query = query!.Trim('"', '\''); + + // Special characters become dashes + query = query.Replace(' ', '-'); + query = query.Replace('/', '-'); + query = query.Replace('\\', '/'); + + // Lowercase is defined per language + query = query.ToLowerInvariant(); + + // Keep getting quicksearch pages until there are none left + try + { + int pageNumber = 1; + while (true) + { + List pageIds = await rc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++)); + ids.AddRange(pageIds); + if (pageIds.Count <= 1) + break; + } + } + catch (Exception ex) + { + Console.WriteLine($"An exception occurred while trying to log in: {ex}"); + return null; + } + + return ids; + } + + /// + /// Download the disc pages associated with a given quicksearch query + /// + /// RedumpClient for connectivity + /// Query string to attempt to search for + /// Output directory to save data to + public static async Task DownloadSearchResults(RedumpClient rc, string? query, string? outDir) + { + // If the query is invalid + if (string.IsNullOrEmpty(query)) + return false; + + // Strip quotes + query = query!.Trim('"', '\''); + + // Special characters become dashes + query = query.Replace(' ', '-'); + query = query.Replace('/', '-'); + query = query.Replace('\\', '/'); + + // Lowercase is defined per language + query = query.ToLowerInvariant(); + + // Keep getting quicksearch pages until there are none left + int pageNumber = 1; + while (true) + { + if (!await rc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++), outDir, false)) + break; + } + + return true; + } + } +} \ No newline at end of file diff --git a/SabreTools.RedumpLib/Web/User.cs b/SabreTools.RedumpLib/Web/User.cs new file mode 100644 index 0000000..cb6a6a5 --- /dev/null +++ b/SabreTools.RedumpLib/Web/User.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using SabreTools.RedumpLib.Data; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Contains logic for dealing with users + /// + public static class User + { + /// + /// Download the disc pages associated with the given user + /// + /// RedumpClient for connectivity + /// Username to check discs for + /// Output directory to save data to + public static async Task DownloadUser(RedumpClient rc, string? username, string? outDir) + { + if (!rc.LoggedIn || string.IsNullOrEmpty(username)) + { + Console.WriteLine("User download functionality is only available to Redump members"); + return false; + } + + // Keep getting user pages until there are none left + int pageNumber = 1; + while (true) + { + if (!await rc.CheckSingleSitePage(string.Format(Constants.UserDumpsUrl, username, pageNumber++), outDir, false)) + break; + } + + return true; + } + + /// + /// Download the last modified disc pages associated with the given user, until first failure + /// + /// RedumpClient for connectivity + /// Username to check discs for + /// Output directory to save data to + public static async Task DownloadUserLastModified(RedumpClient rc, string? username, string? outDir) + { + if (!rc.LoggedIn || string.IsNullOrEmpty(username)) + { + Console.WriteLine("User download functionality is only available to Redump members"); + return false; + } + + // Keep getting last modified user pages until there are none left + int pageNumber = 1; + while (true) + { + if (!await rc.CheckSingleSitePage(string.Format(Constants.UserDumpsLastModifiedUrl, username, pageNumber++), outDir, true)) + break; + } + + return true; + } + + /// + /// List the disc IDs associated with the given user + /// + /// RedumpClient for connectivity + /// Username to check discs for + /// All disc IDs for the given user, null on error + public static async Task?> ListUser(RedumpClient rc, string? username) + { + List ids = []; + + if (!rc.LoggedIn || string.IsNullOrEmpty(username)) + { + Console.WriteLine("User download functionality is only available to Redump members"); + return ids; + } + + // Keep getting user pages until there are none left + try + { + int pageNumber = 1; + while (true) + { + List pageIds = await rc.CheckSingleSitePage(string.Format(Constants.UserDumpsUrl, username, pageNumber++)); + ids.AddRange(pageIds); + if (pageIds.Count <= 1) + break; + } + } + catch (Exception ex) + { + Console.WriteLine($"An exception occurred while trying to log in: {ex}"); + return null; + } + + return ids; + } + } +} \ No newline at end of file diff --git a/SabreTools.RedumpLib/Web/WIP.cs b/SabreTools.RedumpLib/Web/WIP.cs new file mode 100644 index 0000000..0b2009a --- /dev/null +++ b/SabreTools.RedumpLib/Web/WIP.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading.Tasks; +using SabreTools.RedumpLib.Data; + +namespace SabreTools.RedumpLib.Web +{ + /// + /// Contains logic for dealing with WIP queue + /// + public static class WIP + { + /// + /// Download the last submitted WIP disc pages + /// + /// RedumpClient for connectivity + /// Output directory to save data to + public static async Task DownloadLastSubmitted(RedumpClient rc, string? outDir) + { + return await rc.CheckSingleWIPPage(Constants.WipDumpsUrl, outDir, false); + } + + /// + /// Download the specified range of WIP disc pages + /// + /// RedumpClient for connectivity + /// Output directory to save data to + /// Starting ID for the range + /// Ending ID for the range (inclusive) + public static async Task DownloadWIPRange(RedumpClient rc, string? outDir, int minId = 0, int maxId = 0) + { + if (!rc.LoggedIn || !rc.IsStaff) + { + Console.WriteLine("WIP download functionality is only available to Redump moderators"); + return false; + } + + for (int id = minId; id <= maxId; id++) + { + if (await rc.DownloadSingleWIPID(id, outDir, true)) + DelayHelper.DelayRandom(); // Intentional sleep here so we don't flood the server + } + + return true; + } + } +} \ No newline at end of file