From 1b682530899cec8a57e5c40a3bd2995fba1e2ae5 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 27 Jun 2024 01:41:17 -0400 Subject: [PATCH] Remove now-redundant classes --- Web/RedumpHttpClient.cs | 831 ---------------------------------------- Web/RedumpWebClient.cs | 797 -------------------------------------- 2 files changed, 1628 deletions(-) delete mode 100644 Web/RedumpHttpClient.cs delete mode 100644 Web/RedumpWebClient.cs diff --git a/Web/RedumpHttpClient.cs b/Web/RedumpHttpClient.cs deleted file mode 100644 index b7e6c47..0000000 --- a/Web/RedumpHttpClient.cs +++ /dev/null @@ -1,831 +0,0 @@ -#if NETCOREAPP - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using SabreTools.RedumpLib.Data; - -namespace SabreTools.RedumpLib.Web -{ - public class RedumpHttpClient : HttpClient - { - #region Properties - - /// - /// Determines if user is logged into Redump - /// - public bool LoggedIn { get; private set; } = false; - - /// - /// Determines if the user is a staff member - /// - public bool IsStaff { get; private set; } = false; - - /// - /// Maximum retry count for any operation - /// - public int RetryCount { get; private set; } = 3; - - #endregion - - /// - /// Constructor - /// - public RedumpHttpClient() - : base(new HttpClientHandler { UseCookies = true }) - { - } - - #region Credentials - - /// - /// Validate supplied credentials - /// - public async static Task<(bool?, string?)> ValidateCredentials(string username, string password) - { - // If options are invalid or we're missing something key, just return - if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password)) - return (false, null); - - // Try logging in with the supplied credentials otherwise - using RedumpHttpClient httpClient = new(); - - bool? loggedIn = await httpClient.Login(username, password); - if (loggedIn == true) - return (true, "Redump username and password accepted!"); - else if (loggedIn == false) - return (false, "Redump username and password denied!"); - else - return (null, "An error occurred validating your credentials!"); - } - - /// - /// Login to Redump, if possible - /// - /// Redump username - /// Redump password - /// True if the user could be logged in, false otherwise, null on error - public async Task Login(string username, string password) - { - // Credentials verification - if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) - { - Console.WriteLine("Credentials entered, will attempt Redump login..."); - } - else if (!string.IsNullOrEmpty(username) && string.IsNullOrEmpty(password)) - { - Console.WriteLine("Only a username was specified, will not attempt Redump login..."); - return false; - } - else if (string.IsNullOrEmpty(username)) - { - Console.WriteLine("No credentials entered, will not attempt Redump login..."); - return false; - } - - // HTTP encode the password - password = WebUtility.UrlEncode(password); - - // Attempt to login up to 3 times - for (int i = 0; i < 3; i++) - { - try - { - // Get the current token from the login page - var loginPage = await DownloadStringWithRetries(Constants.LoginUrl); - string token = Constants.TokenRegex.Match(loginPage ?? string.Empty).Groups[1].Value; - - // Construct the login request - var postContent = new StringContent($"form_sent=1&redirect_url=&csrf_token={token}&req_username={username}&req_password={password}&save_pass=0", Encoding.UTF8); - postContent.Headers.ContentType = MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded"); - - // Send the login request and get the result - var response = await PostAsync(Constants.LoginUrl, postContent); - string? responseContent = null; - if (response?.Content != null) - responseContent = await response.Content.ReadAsStringAsync(); - - if (string.IsNullOrEmpty(responseContent)) - { - Console.WriteLine($"An error occurred while trying to log in on attempt {i}: No response"); - continue; - } - - if (responseContent.Contains("Incorrect username and/or password.")) - { - Console.WriteLine("Invalid credentials entered, continuing without logging in..."); - return false; - } - - // The user was able to be logged in - Console.WriteLine("Credentials accepted! Logged into Redump..."); - LoggedIn = true; - - // If the user is a moderator or staff, set accordingly - if (responseContent.Contains("http://forum.redump.org/forum/9/staff/")) - IsStaff = true; - - return true; - } - catch (Exception ex) - { - Console.WriteLine($"An exception occurred while trying to log in on attempt {i}: {ex}"); - } - } - - Console.WriteLine("Could not login to Redump in 3 attempts, continuing without logging in..."); - return false; - } - - #endregion - - #region Single Page Helpers - - /// - /// Process a Redump site page as a list of possible IDs or disc page - /// - /// Base URL to download using - /// List of IDs from the page, empty on error - public async Task> CheckSingleSitePage(string url) - { - List ids = []; - - // Try to retrieve the data - string? dumpsPage = await DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return ids; - - // If we have a single disc page already - if (dumpsPage.Contains("Download:")) - { - var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; - if (int.TryParse(value, out int id)) - ids.Add(id); - - return ids; - } - - // Otherwise, traverse each dump on the page - var matches = Constants.DiscRegex.Matches(dumpsPage); - foreach (Match? match in matches.Cast()) - { - if (match == null) - continue; - - try - { - if (int.TryParse(match.Groups[1].Value, out int value)) - ids.Add(value); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return ids; - } - - /// - /// Process a Redump site page as a list of possible IDs or disc page - /// - /// Base URL to download using - /// Output directory to save data to - /// True to return on first error, false otherwise - /// True if the page could be downloaded, false otherwise - public async Task CheckSingleSitePage(string url, string? outDir, bool failOnSingle) - { - // Try to retrieve the data - string? dumpsPage = await DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return false; - - // If we have a single disc page already - if (dumpsPage.Contains("Download:")) - { - var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; - if (int.TryParse(value, out int id)) - { - bool downloaded = await DownloadSingleSiteID(id, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - - return false; - } - - // Otherwise, traverse each dump on the page - var matches = Constants.DiscRegex.Matches(dumpsPage); - foreach (Match? match in matches.Cast()) - { - if (match == null) - continue; - - try - { - if (int.TryParse(match.Groups[1].Value, out int value)) - { - bool downloaded = await DownloadSingleSiteID(value, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return true; - } - - /// - /// Process a Redump WIP page as a list of possible IDs or disc page - /// - /// RedumpWebClient to access the packs - /// List of IDs from the page, empty on error - public async Task> CheckSingleWIPPage(string url) - { - List ids = []; - - // Try to retrieve the data - string? dumpsPage = await DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return ids; - - // Otherwise, traverse each dump on the page - var matches = Constants.NewDiscRegex.Matches(dumpsPage); - foreach (Match? match in matches.Cast()) - { - if (match == null) - continue; - - try - { - if (int.TryParse(match.Groups[2].Value, out int value)) - ids.Add(value); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return ids; - } - - /// - /// Process a Redump WIP page as a list of possible IDs or disc page - /// - /// RedumpWebClient to access the packs - /// Output directory to save data to - /// True to return on first error, false otherwise - /// True if the page could be downloaded, false otherwise - public async Task CheckSingleWIPPage(string url, string? outDir, bool failOnSingle) - { - // Try to retrieve the data - string? dumpsPage = await DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return false; - - // Otherwise, traverse each dump on the page - var matches = Constants.NewDiscRegex.Matches(dumpsPage); - foreach (Match? match in matches.Cast()) - { - if (match == null) - continue; - - try - { - if (int.TryParse(match.Groups[2].Value, out int value)) - { - bool downloaded = await DownloadSingleWIPID(value, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return true; - } - - #endregion - - #region Download Helpers - - /// - /// Download a single pack - /// - /// Base URL to download using - /// System to download packs for - /// Byte array containing the downloaded pack, null on error - public async Task DownloadSinglePack(string url, RedumpSystem? system) - { - try - { - return await GetByteArrayAsync(string.Format(url, system.ShortName())); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download a single pack - /// - /// Base URL to download using - /// System to download packs for - /// Output directory to save data to - /// Named subfolder for the pack, used optionally - public async Task DownloadSinglePack(string url, RedumpSystem? system, string? outDir, string? subfolder) - { - try - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string tempfile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - string packUri = string.Format(url, system.ShortName()); - - // Make the call to get the pack - string? remoteFileName = await DownloadFile(packUri, tempfile); - MoveOrDelete(tempfile, remoteFileName, outDir!, subfolder); - return true; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return false; - } - } - - /// - /// Download an individual site ID data, if possible - /// - /// Redump disc ID to retrieve - /// String containing the page contents if successful, null on error - public async Task DownloadSingleSiteID(int id) - { - string paddedId = id.ToString().PadLeft(6, '0'); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.DiscPageUrl, +id); - string? discPage = await DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) - { - Console.WriteLine($"ID {paddedId} could not be found!"); - return null; - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return discPage; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download an individual site ID data, if possible - /// - /// Redump disc ID to retrieve - /// Output directory to save data to - /// True to rename deleted entries, false otherwise - /// True if all data was downloaded, false otherwise - public async Task DownloadSingleSiteID(int id, string? outDir, bool rename) - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string paddedId = id.ToString().PadLeft(6, '0'); - string paddedIdDir = Path.Combine(outDir, paddedId); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.DiscPageUrl, +id); - string? discPage = await DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) - { - try - { - if (rename) - { - if (Directory.Exists(paddedIdDir) && rename) - Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); - else - Directory.CreateDirectory(paddedIdDir + "-deleted"); - } - } - catch { } - - Console.WriteLine($"ID {paddedId} could not be found!"); - return false; - } - - // Check if the page has been updated since the last time it was downloaded, if possible - if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) - { - // Read in the cached file - var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); - - // Check for the last modified date in both pages - var oldResult = Constants.LastModifiedRegex.Match(oldDiscPage); - var newResult = Constants.LastModifiedRegex.Match(discPage); - - // If both pages contain the same modified date, skip it - if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - - // If neither page contains a modified date, skip it - else if (!oldResult.Success && !newResult.Success) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - } - - // Create ID subdirectory - Directory.CreateDirectory(paddedIdDir); - - // View Edit History - if (discPage.Contains($" - /// Download an individual WIP ID data, if possible - /// - /// Redump WIP disc ID to retrieve - /// String containing the page contents if successful, null on error - public async Task DownloadSingleWIPID(int id) - { - string paddedId = id.ToString().PadLeft(6, '0'); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); - string? discPage = await DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) - { - Console.WriteLine($"ID {paddedId} could not be found!"); - return null; - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return discPage; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download an individual WIP ID data, if possible - /// - /// Redump WIP disc ID to retrieve - /// Output directory to save data to - /// True to rename deleted entries, false otherwise - /// True if all data was downloaded, false otherwise - public async Task DownloadSingleWIPID(int id, string? outDir, bool rename) - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string paddedId = id.ToString().PadLeft(6, '0'); - string paddedIdDir = Path.Combine(outDir, paddedId); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); - string? discPage = await DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) - { - try - { - if (rename) - { - if (Directory.Exists(paddedIdDir) && rename) - Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); - else - Directory.CreateDirectory(paddedIdDir + "-deleted"); - } - } - catch { } - - Console.WriteLine($"ID {paddedId} could not be found!"); - return false; - } - - // Check if the page has been updated since the last time it was downloaded, if possible - if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) - { - // Read in the cached file - var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); - - // Check for the full match ID in both pages - var oldResult = Constants.FullMatchRegex.Match(oldDiscPage); - var newResult = Constants.FullMatchRegex.Match(discPage); - - // If both pages contain the same ID, skip it - if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - - // If neither page contains an ID, skip it - else if (!oldResult.Success && !newResult.Success) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - } - - // Create ID subdirectory - Directory.CreateDirectory(paddedIdDir); - - // HTML - using (var discStreamWriter = File.CreateText(Path.Combine(paddedIdDir, "disc.html"))) - { - discStreamWriter.Write(discPage); - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return true; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return false; - } - } - - #endregion - - #region Helpers - - /// - /// Download a set of packs - /// - /// Base URL to download using - /// Systems to download packs for - /// Name of the pack that is downloading - public async Task> DownloadPacks(string url, RedumpSystem?[] systems, string title) - { - var packsDictionary = new Dictionary(); - - Console.WriteLine($"Downloading {title}"); - foreach (var system in systems) - { - // If the system is invalid, we can't do anything - if (system == null || !system.IsAvailable()) - continue; - - // If we didn't have credentials - if (!LoggedIn && system.IsBanned()) - continue; - - // If the system is unknown, we can't do anything - string? longName = system.LongName(); - if (string.IsNullOrEmpty(longName)) - continue; - - Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}"); - byte[]? pack = await DownloadSinglePack(url, system); - if (pack != null) - packsDictionary.Add(system.Value, pack); - } - - Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); - Console.WriteLine(); - - return packsDictionary; - } - - /// - /// Download a set of packs - /// - /// Base URL to download using - /// Systems to download packs for - /// Name of the pack that is downloading - /// Output directory to save data to - /// Named subfolder for the pack, used optionally - public async Task DownloadPacks(string url, RedumpSystem?[] systems, string title, string? outDir, string? subfolder) - { - Console.WriteLine($"Downloading {title}"); - foreach (var system in systems) - { - // If the system is invalid, we can't do anything - if (system == null || !system.IsAvailable()) - continue; - - // If we didn't have credentials - if (!LoggedIn && system.IsBanned()) - continue; - - // If the system is unknown, we can't do anything - string? longName = system.LongName(); - if (string.IsNullOrEmpty(longName)) - continue; - - Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}"); - await DownloadSinglePack(url, system, outDir, subfolder); - } - - Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); - Console.WriteLine(); - return true; - } - - /// - /// Download from a URI to a local file - /// - /// Remote URI to retrieve - /// Filename to write to - /// The remote filename from the URI, null on error - private async Task DownloadFile(string uri, string fileName) - { - // Make the call to get the file - var response = await GetAsync(uri); - if (response?.Content?.Headers == null || !response.IsSuccessStatusCode) - { - Console.WriteLine($"Could not download {uri}"); - return null; - } - - // Copy the data to a local temp file - using (var responseStream = await response.Content.ReadAsStreamAsync()) - using (var tempFileStream = File.OpenWrite(fileName)) - { - responseStream.CopyTo(tempFileStream); - } - - return response.Content.Headers.ContentDisposition?.FileName?.Replace("\"", ""); - } - - /// - /// Download from a URI to a string - /// - /// Remote URI to retrieve - /// String from the URI, null on error - private async Task DownloadStringWithRetries(string uri) - { - // Only retry a positive number of times - if (RetryCount <= 0) - return null; - - for (int i = 0; i < RetryCount; i++) - { - try - { - return await GetStringAsync(uri); - } - catch { } - - // Sleep for 100ms if the last attempt failed - Thread.Sleep(100); - } - - return null; - } - - /// - /// Move a tempfile to a new name unless it aleady exists, in which case, delete the tempfile - /// - /// Path to existing temporary file - /// Path to new output file - /// Output directory to save data to - /// Optional subfolder to append to the path - private static void MoveOrDelete(string tempfile, string? newfile, string outDir, string? subfolder) - { - // If we don't have a file to move to, just delete the temp file - if (string.IsNullOrEmpty(newfile)) - { - File.Delete(tempfile); - return; - } - - // If we have a subfolder, create it and update the newfile name - if (!string.IsNullOrEmpty(subfolder)) - { - if (!Directory.Exists(Path.Combine(outDir, subfolder))) - Directory.CreateDirectory(Path.Combine(outDir, subfolder)); - - newfile = Path.Combine(subfolder, newfile); - } - - // If the file already exists, don't overwrite it - if (File.Exists(Path.Combine(outDir, newfile))) - File.Delete(tempfile); - else - File.Move(tempfile, Path.Combine(outDir, newfile)); - } - - #endregion - } -} - -#endif \ No newline at end of file diff --git a/Web/RedumpWebClient.cs b/Web/RedumpWebClient.cs deleted file mode 100644 index fb3101a..0000000 --- a/Web/RedumpWebClient.cs +++ /dev/null @@ -1,797 +0,0 @@ -#if NETFRAMEWORK - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using SabreTools.RedumpLib.Data; - -namespace SabreTools.RedumpLib.Web -{ - // https://stackoverflow.com/questions/1777221/using-cookiecontainer-with-webclient-class - public class RedumpWebClient : WebClient - { - #region Properties - - private readonly CookieContainer m_container = new(); - - /// - /// Determines if user is logged into Redump - /// - public bool LoggedIn { get; private set; } = false; - - /// - /// Determines if the user is a staff member - /// - public bool IsStaff { get; private set; } = false; - - /// - /// Maximum retry count for any operation - /// - public int RetryCount { get; private set; } = 3; - - #endregion - - /// - /// Get the last downloaded filename, if possible - /// - /// - public string? GetLastFilename() - { - // If the response headers are null or empty - if (ResponseHeaders == null || ResponseHeaders.Count == 0) - return null; - - // If we don't have the response header we care about - string headerValue = ResponseHeaders.Get("Content-Disposition"); - if (string.IsNullOrEmpty(headerValue)) - return null; - - // Extract the filename from the value - return headerValue.Substring(headerValue.IndexOf("filename=") + 9).Replace("\"", ""); - } - - /// - protected override WebRequest GetWebRequest(Uri address) - { - WebRequest request = base.GetWebRequest(address); - if (request is HttpWebRequest webRequest) - webRequest.CookieContainer = m_container; - - return request; - } - - #region Credentials - - /// - /// Validate supplied credentials - /// - public static (bool?, string?) ValidateCredentials(string username, string password) - { - // If options are invalid or we're missing something key, just return - if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password)) - return (false, null); - - // Try logging in with the supplied credentials otherwise - using RedumpWebClient wc = new(); - bool? loggedIn = wc.Login(username, password); - if (loggedIn == true) - return (true, "Redump username and password accepted!"); - else if (loggedIn == false) - return (false, "Redump username and password denied!"); - else - return (null, "An error occurred validating your credentials!"); - } - - /// - /// Login to Redump, if possible - /// - /// Redump username - /// Redump password - /// True if the user could be logged in, false otherwise, null on error - public bool? Login(string username, string password) - { - // Credentials verification - if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) - { - Console.WriteLine("Credentials entered, will attempt Redump login..."); - } - else if (!string.IsNullOrEmpty(username) && string.IsNullOrEmpty(password)) - { - Console.WriteLine("Only a username was specified, will not attempt Redump login..."); - return false; - } - else if (string.IsNullOrEmpty(username)) - { - Console.WriteLine("No credentials entered, will not attempt Redump login..."); - return false; - } - - // HTTP encode the password -#if NET20 || NET35 || NET40 - password = Uri.EscapeUriString(password); -#else - password = WebUtility.UrlEncode(password); -#endif - - // Attempt to login up to 3 times - for (int i = 0; i < 3; i++) - { - try - { - // Get the current token from the login page - var loginPage = DownloadStringWithRetries(Constants.LoginUrl); - string token = Constants.TokenRegex.Match(loginPage ?? string.Empty).Groups[1].Value; - - // Construct the login request - Headers[HttpRequestHeader.ContentType] = "application/x-www-form-urlencoded"; - Encoding = Encoding.UTF8; - var response = UploadString(Constants.LoginUrl, $"form_sent=1&redirect_url=&csrf_token={token}&req_username={username}&req_password={password}&save_pass=0"); - - if (response.Contains("Incorrect username and/or password.")) - { - Console.WriteLine("Invalid credentials entered, continuing without logging in..."); - return false; - } - - // The user was able to be logged in - Console.WriteLine("Credentials accepted! Logged into Redump..."); - LoggedIn = true; - - // If the user is a moderator or staff, set accordingly - if (response.Contains("http://forum.redump.org/forum/9/staff/")) - IsStaff = true; - - return true; - } - catch (Exception ex) - { - Console.WriteLine($"An exception occurred while trying to log in on attempt {i}: {ex}"); - } - } - - Console.WriteLine("Could not login to Redump in 3 attempts, continuing without logging in..."); - return false; - } - - #endregion - - #region Single Page Helpers - - /// - /// Process a Redump site page as a list of possible IDs or disc page - /// - /// Base URL to download using - /// List of IDs from the page, empty on error - public List CheckSingleSitePage(string url) - { - List ids = []; - - // Try to retrieve the data - string? dumpsPage = DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return ids; - - // If we have a single disc page already - if (dumpsPage.Contains("Download:")) - { - var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; - if (int.TryParse(value, out int id)) - ids.Add(id); - - return ids; - } - - // Otherwise, traverse each dump on the page - var matches = Constants.DiscRegex.Matches(dumpsPage); - foreach (Match match in matches) - { - try - { - if (int.TryParse(match.Groups[1].Value, out int value)) - ids.Add(value); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return ids; - } - - /// - /// Process a Redump site page as a list of possible IDs or disc page - /// - /// Base URL to download using - /// Output directory to save data to - /// True to return on first error, false otherwise - /// True if the page could be downloaded, false otherwise - public bool CheckSingleSitePage(string url, string? outDir, bool failOnSingle) - { - // Try to retrieve the data - string? dumpsPage = DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return false; - - // If we have a single disc page already - if (dumpsPage.Contains("Download:")) - { - var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; - if (int.TryParse(value, out int id)) - { - bool downloaded = DownloadSingleSiteID(id, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - - return false; - } - - // Otherwise, traverse each dump on the page - var matches = Constants.DiscRegex.Matches(dumpsPage); - foreach (Match match in matches) - { - try - { - if (int.TryParse(match.Groups[1].Value, out int value)) - { - bool downloaded = DownloadSingleSiteID(value, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return true; - } - - /// - /// Process a Redump WIP page as a list of possible IDs or disc page - /// - /// RedumpWebClient to access the packs - /// List of IDs from the page, empty on error - public List CheckSingleWIPPage(string url) - { - List ids = []; - - // Try to retrieve the data - string? dumpsPage = DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return ids; - - // Otherwise, traverse each dump on the page - var matches = Constants.NewDiscRegex.Matches(dumpsPage); - foreach (Match match in matches) - { - try - { - if (int.TryParse(match.Groups[2].Value, out int value)) - ids.Add(value); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return ids; - } - - /// - /// Process a Redump WIP page as a list of possible IDs or disc page - /// - /// RedumpWebClient to access the packs - /// Output directory to save data to - /// True to return on first error, false otherwise - /// True if the page could be downloaded, false otherwise - public bool CheckSingleWIPPage(string url, string? outDir, bool failOnSingle) - { - // Try to retrieve the data - string? dumpsPage = DownloadStringWithRetries(url); - - // If we have no dumps left - if (dumpsPage == null || dumpsPage.Contains("No discs found.")) - return false; - - // Otherwise, traverse each dump on the page - var matches = Constants.NewDiscRegex.Matches(dumpsPage); - foreach (Match match in matches) - { - try - { - if (int.TryParse(match.Groups[2].Value, out int value)) - { - bool downloaded = DownloadSingleWIPID(value, outDir, false); - if (!downloaded && failOnSingle) - return false; - } - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - continue; - } - } - - return true; - } - - #endregion - - #region Download Helpers - - /// - /// Download a single pack - /// - /// Base URL to download using - /// System to download packs for - /// Byte array containing the downloaded pack, null on error - public byte[]? DownloadSinglePack(string url, RedumpSystem? system) - { - try - { - return DownloadData(string.Format(url, system.ShortName())); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download a single pack - /// - /// Base URL to download using - /// System to download packs for - /// Output directory to save data to - /// Named subfolder for the pack, used optionally - public void DownloadSinglePack(string url, RedumpSystem? system, string? outDir, string? subfolder) - { - try - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string tempfile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - DownloadFile(string.Format(url, system.ShortName()), tempfile); - MoveOrDelete(tempfile, GetLastFilename(), outDir!, subfolder); - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - } - } - - /// - /// Download an individual site ID data, if possible - /// - /// Redump disc ID to retrieve - /// String containing the page contents if successful, null on error - public string? DownloadSingleSiteID(int id) - { - string paddedId = id.ToString().PadLeft(6, '0'); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.DiscPageUrl, +id); - string? discPage = DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) - { - Console.WriteLine($"ID {paddedId} could not be found!"); - return null; - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return discPage; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download an individual site ID data, if possible - /// - /// Redump disc ID to retrieve - /// Output directory to save data to - /// True to rename deleted entries, false otherwise - /// True if all data was downloaded, false otherwise - public bool DownloadSingleSiteID(int id, string? outDir, bool rename) - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string paddedId = id.ToString().PadLeft(6, '0'); - string paddedIdDir = Path.Combine(outDir, paddedId); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.DiscPageUrl, +id); - string? discPage = DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) - { - try - { - if (rename) - { - if (Directory.Exists(paddedIdDir) && rename) - Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); - else - Directory.CreateDirectory(paddedIdDir + "-deleted"); - } - } - catch { } - - Console.WriteLine($"ID {paddedId} could not be found!"); - return false; - } - - // Check if the page has been updated since the last time it was downloaded, if possible - if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) - { - // Read in the cached file - var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); - - // Check for the last modified date in both pages - var oldResult = Constants.LastModifiedRegex.Match(oldDiscPage); - var newResult = Constants.LastModifiedRegex.Match(discPage); - - // If both pages contain the same modified date, skip it - if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - - // If neither page contains a modified date, skip it - else if (!oldResult.Success && !newResult.Success) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - } - - // Create ID subdirectory - Directory.CreateDirectory(paddedIdDir); - - // View Edit History - if (discPage.Contains($" - /// Download an individual WIP ID data, if possible - /// - /// Redump WIP disc ID to retrieve - /// String containing the page contents if successful, null on error - public string? DownloadSingleWIPID(int id) - { - string paddedId = id.ToString().PadLeft(6, '0'); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); - string? discPage = DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) - { - Console.WriteLine($"ID {paddedId} could not be found!"); - return null; - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return discPage; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return null; - } - } - - /// - /// Download an individual WIP ID data, if possible - /// - /// Redump WIP disc ID to retrieve - /// Output directory to save data to - /// True to rename deleted entries, false otherwise - /// True if all data was downloaded, false otherwise - public bool DownloadSingleWIPID(int id, string? outDir, bool rename) - { - // If no output directory is defined, use the current directory instead - if (string.IsNullOrEmpty(outDir)) - outDir = Environment.CurrentDirectory; - - string paddedId = id.ToString().PadLeft(6, '0'); - string paddedIdDir = Path.Combine(outDir, paddedId); - Console.WriteLine($"Processing ID: {paddedId}"); - try - { - // Try to retrieve the data - string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); - string? discPage = DownloadStringWithRetries(discPageUri); - - if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) - { - try - { - if (rename) - { - if (Directory.Exists(paddedIdDir) && rename) - Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); - else - Directory.CreateDirectory(paddedIdDir + "-deleted"); - } - } - catch { } - - Console.WriteLine($"ID {paddedId} could not be found!"); - return false; - } - - // Check if the page has been updated since the last time it was downloaded, if possible - if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) - { - // Read in the cached file - var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); - - // Check for the full match ID in both pages - var oldResult = Constants.FullMatchRegex.Match(oldDiscPage); - var newResult = Constants.FullMatchRegex.Match(discPage); - - // If both pages contain the same ID, skip it - if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - - // If neither page contains an ID, skip it - else if (!oldResult.Success && !newResult.Success) - { - Console.WriteLine($"ID {paddedId} has not been changed since last download"); - return false; - } - } - - // Create ID subdirectory - Directory.CreateDirectory(paddedIdDir); - - // HTML - using (var discStreamWriter = File.CreateText(Path.Combine(paddedIdDir, "disc.html"))) - { - discStreamWriter.Write(discPage); - } - - Console.WriteLine($"ID {paddedId} has been successfully downloaded"); - return true; - } - catch (Exception ex) - { - Console.WriteLine($"An exception has occurred: {ex}"); - return false; - } - } - - #endregion - - #region Helpers - - /// - /// Download a set of packs - /// - /// Base URL to download using - /// Systems to download packs for - /// Name of the pack that is downloading - public Dictionary DownloadPacks(string url, RedumpSystem?[] systems, string title) - { - var packsDictionary = new Dictionary(); - - Console.WriteLine($"Downloading {title}"); - foreach (var system in systems) - { - // If the system is invalid, we can't do anything - if (system == null || !system.IsAvailable()) - continue; - - // If we didn't have credentials - if (!LoggedIn && system.IsBanned()) - continue; - - // If the system is unknown, we can't do anything - string? longName = system.LongName(); - if (string.IsNullOrEmpty(longName)) - continue; - - Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName!.Length - 1)}"); - byte[]? pack = DownloadSinglePack(url, system); - if (pack != null) - packsDictionary.Add(system.Value, pack); - } - - Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); - Console.WriteLine(); - - return packsDictionary; - } - - /// - /// Download a set of packs - /// - /// Base URL to download using - /// Systems to download packs for - /// Name of the pack that is downloading - /// Output directory to save data to - /// Named subfolder for the pack, used optionally - public void DownloadPacks(string url, RedumpSystem?[] systems, string title, string? outDir, string? subfolder) - { - Console.WriteLine($"Downloading {title}"); - foreach (var system in systems) - { - // If the system is invalid, we can't do anything - if (system == null || !system.IsAvailable()) - continue; - - // If we didn't have credentials - if (!LoggedIn && system.IsBanned()) - continue; - - // If the system is unknown, we can't do anything - string? longName = system.LongName(); - if (string.IsNullOrEmpty(longName)) - continue; - - Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName!.Length - 1)}"); - DownloadSinglePack(url, system, outDir, subfolder); - } - - Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); - Console.WriteLine(); - } - - /// - /// Download from a URI to a string - /// - /// Remote URI to retrieve - /// String from the URI, null on error - private string? DownloadStringWithRetries(string uri) - { - // Only retry a positive number of times - if (RetryCount <= 0) - return null; - - for (int i = 0; i < RetryCount; i++) - { - try - { - return DownloadString(uri); - } - catch { } - - // Sleep for 100ms if the last attempt failed - Thread.Sleep(100); - } - - return null; - } - - /// - /// Move a tempfile to a new name unless it aleady exists, in which case, delete the tempfile - /// - /// Path to existing temporary file - /// Path to new output file - /// Output directory to save data to - /// Optional subfolder to append to the path - private static void MoveOrDelete(string tempfile, string? newfile, string outDir, string? subfolder) - { - if (!string.IsNullOrEmpty(newfile)) - { - if (!string.IsNullOrEmpty(subfolder)) - { - if (!Directory.Exists(Path.Combine(outDir, subfolder))) - Directory.CreateDirectory(Path.Combine(outDir, subfolder)); - - newfile = Path.Combine(subfolder, newfile); - } - - if (File.Exists(Path.Combine(outDir, newfile))) - File.Delete(tempfile); - else - File.Move(tempfile, Path.Combine(outDir, newfile)); - } - else - { - File.Delete(tempfile); - } - } - - #endregion - } -} - -#endif \ No newline at end of file