diff --git a/CHANGELIST.md b/CHANGELIST.md index 36ab849c..b0597ffb 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -99,6 +99,7 @@ - Fix .bin file paths; update internal filename generation - Disable nonstandard BD-ROM sizes - Trim leading file paths for XBONE +- Give .NET 6 priority for web calls ### 2.3 (2022-02-05) - Start overhauling Redump information pulling, again diff --git a/MPF.Check/Program.cs b/MPF.Check/Program.cs index 8f934c34..b59d8c6c 100644 --- a/MPF.Check/Program.cs +++ b/MPF.Check/Program.cs @@ -33,7 +33,11 @@ namespace MPF.Check protectionProgress.ProgressChanged += ProgressUpdated; // Validate the supplied credentials +#if NET48 || NETSTANDARD2_1 (bool? _, string message) = RedumpWebClient.ValidateCredentials(options?.RedumpUsername, options?.RedumpPassword); +#else + (bool? _, string message) = RedumpHttpClient.ValidateCredentials(options?.RedumpUsername, options?.RedumpPassword).ConfigureAwait(false).GetAwaiter().GetResult(); +#endif if (!string.IsNullOrWhiteSpace(message)) Console.WriteLine(message); diff --git a/MPF.Library/InfoTool.cs b/MPF.Library/InfoTool.cs index 28611aaa..a6b92a72 100644 --- a/MPF.Library/InfoTool.cs +++ b/MPF.Library/InfoTool.cs @@ -97,7 +97,11 @@ namespace MPF.Library // Get a list of matching IDs for each line in the DAT if (!string.IsNullOrEmpty(info.TracksAndWriteOffsets.ClrMameProData) && options.HasRedumpLogin) +#if NET48 || NETSTANDARD2_1 FillFromRedump(options, info, resultProgress); +#else + _ = await FillFromRedump(options, info, resultProgress); +#endif // If we have both ClrMamePro and Size and Checksums data, remove the ClrMamePro if (!string.IsNullOrWhiteSpace(info.SizeAndChecksums.CRC32)) @@ -561,9 +565,9 @@ namespace MPF.Library } } - #endregion +#endregion - #region Information Output +#region Information Output /// /// Compress log files to save space @@ -1134,9 +1138,9 @@ namespace MPF.Library AddIfExists(output, key, string.Join(", ", value.Select(o => o.ToString())), indent); } - #endregion +#endregion - #region Normalization +#region Normalization /// /// Adjust the disc type based on size and layerbreak information @@ -1257,9 +1261,9 @@ namespace MPF.Library return (directory, filename); } - #endregion +#endregion - #region Web Calls +#region Web Calls /// /// Create a new SubmissionInfo object from a disc page @@ -1444,11 +1448,19 @@ namespace MPF.Library /// RedumpWebClient for making the connection /// Existing SubmissionInfo object to fill /// Redump disc ID to retrieve - private static void FillFromId(RedumpWebClient wc, SubmissionInfo info, int id) +#if NET48 || NETSTANDARD2_1 + private static bool FillFromId(RedumpWebClient wc, SubmissionInfo info, int id) { string discData = wc.DownloadSingleSiteID(id); if (string.IsNullOrEmpty(discData)) - return; + return false; +#else + private async static Task FillFromId(RedumpHttpClient wc, SubmissionInfo info, int id) + { + string discData = await wc.DownloadSingleSiteID(id); + if (string.IsNullOrEmpty(discData)) + return false; +#endif // Title, Disc Number/Letter, Disc Title var match = Constants.TitleRegex.Match(discData); @@ -1736,6 +1748,8 @@ namespace MPF.Library else info.LastModified = null; } + + return true; } /// @@ -1744,25 +1758,37 @@ namespace MPF.Library /// Options object representing user-defined options /// Existing SubmissionInfo object to fill /// Optional result progress callback - private static void FillFromRedump(Options options, SubmissionInfo info, IProgress resultProgress = null) +#if NET48 || NETSTANDARD2_1 + private static bool FillFromRedump(Options options, SubmissionInfo info, IProgress resultProgress = null) +#else + private async static Task FillFromRedump(Options options, SubmissionInfo info, IProgress resultProgress = null) +#endif { // Set the current dumper based on username info.DumpersAndStatus.Dumpers = new string[] { options.RedumpUsername }; info.PartiallyMatchedIDs = new List(); +#if NET48 || NETSTANDARD2_1 using (RedumpWebClient wc = new RedumpWebClient()) +#else + using (RedumpHttpClient wc = new RedumpHttpClient()) +#endif { // Login to Redump +#if NET48 || NETSTANDARD2_1 bool? loggedIn = wc.Login(options.RedumpUsername, options.RedumpPassword); +#else + bool? loggedIn = await wc.Login(options.RedumpUsername, options.RedumpPassword); +#endif if (loggedIn == null) { resultProgress?.Report(Result.Failure("There was an unknown error connecting to Redump")); - return; + return false; } else if (loggedIn == false) { // Don't log the as a failure or error - return; + return false; } // Setup the full-track checks @@ -1774,7 +1800,11 @@ namespace MPF.Library string[] splitData = info.TracksAndWriteOffsets.ClrMameProData.Split('\n'); foreach (string hashData in splitData) { +#if NET48 || NETSTANDARD2_1 (bool singleFound, List foundIds) = ValidateSingleTrack(wc, info, hashData, resultProgress); +#else + (bool singleFound, List foundIds) = await ValidateSingleTrack(wc, info, hashData, resultProgress); +#endif // Ensure that all tracks are found allFound &= singleFound; @@ -1806,19 +1836,28 @@ namespace MPF.Library // Exit early if one failed or there are no matched IDs if (!allFound || fullyMatchedIDs.Count == 0) - return; + return false; // Find the first matched ID where the track count matches, we can grab a bunch of info from it int totalMatchedIDsCount = fullyMatchedIDs.Count; for (int i = 0; i < totalMatchedIDsCount; i++) { // Skip if the track count doesn't match +#if NET48 || NETSTANDARD2_1 if (!ValidateTrackCount(wc, fullyMatchedIDs[i], splitData.Length)) continue; +#else + if (!await ValidateTrackCount(wc, fullyMatchedIDs[i], splitData.Length)) + continue; +#endif // Fill in the fields from the existing ID resultProgress?.Report(Result.Success($"Filling fields from existing ID {fullyMatchedIDs[i]}...")); +#if NET48 || NETSTANDARD2_1 FillFromId(wc, info, fullyMatchedIDs[i]); +#else + _ = await FillFromId(wc, info, fullyMatchedIDs[i]); +#endif resultProgress?.Report(Result.Success("Information filling complete!")); // Set the fully matched ID to the current @@ -1835,6 +1874,8 @@ namespace MPF.Library info.PartiallyMatchedIDs.Remove(info.FullyMatchedID.Value); } } + + return true; } /// @@ -1873,7 +1914,11 @@ namespace MPF.Library /// RedumpWebClient for making the connection /// Query string to attempt to search for /// All disc IDs for the given query, null on error +#if NET48 || NETSTANDARD2_1 private static List ListSearchResults(RedumpWebClient wc, string query) +#else + private async static Task> ListSearchResults(RedumpHttpClient wc, string query) +#endif { List ids = new List(); @@ -1894,7 +1939,11 @@ namespace MPF.Library int pageNumber = 1; while (true) { +#if NET48 || NETSTANDARD2_1 List pageIds = wc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++)); +#else + List pageIds = await wc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++)); +#endif ids.AddRange(pageIds); if (pageIds.Count <= 1) break; @@ -1917,7 +1966,11 @@ namespace MPF.Library /// DAT-formatted hash data to parse out /// Optional result progress callback /// True if the track was found, false otherwise; List of found values, if possible +#if NET48 || NETSTANDARD2_1 private static (bool, List) ValidateSingleTrack(RedumpWebClient wc, SubmissionInfo info, string hashData, IProgress resultProgress = null) +#else + private async static Task<(bool, List)> ValidateSingleTrack(RedumpHttpClient wc, SubmissionInfo info, string hashData, IProgress resultProgress = null) +#endif { // If the line isn't parseable, we can't validate if (!GetISOHashValues(hashData, out long _, out string _, out string _, out string sha1)) @@ -1927,7 +1980,11 @@ namespace MPF.Library } // Get all matching IDs for the track +#if NET48 || NETSTANDARD2_1 List newIds = ListSearchResults(wc, sha1); +#else + List newIds = await ListSearchResults(wc, sha1); +#endif // If we got null back, there was an error if (newIds == null) @@ -1956,10 +2013,18 @@ namespace MPF.Library /// Redump disc ID to retrieve /// Local count of tracks for the current disc /// True if the track count matches, false otherwise +#if NET48 || NETSTANDARD2_1 private static bool ValidateTrackCount(RedumpWebClient wc, int id, int localCount) +#else + private async static Task ValidateTrackCount(RedumpHttpClient wc, int id, int localCount) +#endif { // If we can't pull the remote data, we can't match +#if NET48 || NETSTANDARD2_1 string discData = wc.DownloadSingleSiteID(id); +#else + string discData = await wc.DownloadSingleSiteID(id); +#endif if (string.IsNullOrEmpty(discData)) return false; @@ -1978,9 +2043,9 @@ namespace MPF.Library return localCount == remoteCount; } - #endregion +#endregion - #region Helpers +#region Helpers /// /// Format a single site tag to string @@ -2208,6 +2273,6 @@ namespace MPF.Library return sorted; } - #endregion +#endregion } } diff --git a/MPF/ViewModels/OptionsViewModel.cs b/MPF/ViewModels/OptionsViewModel.cs index b4d0e16b..82e0fd4e 100644 --- a/MPF/ViewModels/OptionsViewModel.cs +++ b/MPF/ViewModels/OptionsViewModel.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using System.Windows; using System.Windows.Forms; using MPF.Core.Data; @@ -187,20 +188,30 @@ namespace MPF.UI.ViewModels /// /// Test Redump login credentials /// - private void TestRedumpLogin() +#if NET48 || NETSTANDARD2_1 + private bool? TestRedumpLogin() +#else + private async Task TestRedumpLogin() +#endif { +#if NET48 || NETSTANDARD2_1 (bool? success, string message) = RedumpWebClient.ValidateCredentials(Parent.RedumpUsernameTextBox.Text, Parent.RedumpPasswordBox.Password); +#else + (bool? success, string message) = await RedumpHttpClient.ValidateCredentials(Parent.RedumpUsernameTextBox.Text, Parent.RedumpPasswordBox.Password); +#endif if (success == true) CustomMessageBox.Show(Parent, message, "Success", MessageBoxButton.OK, MessageBoxImage.Information); else if (success == false) CustomMessageBox.Show(Parent, message, "Failure", MessageBoxButton.OK, MessageBoxImage.Error); else CustomMessageBox.Show(Parent, message, "Error", MessageBoxButton.OK, MessageBoxImage.Error); + + return success; } - #endregion +#endregion - #region UI Functionality +#region UI Functionality /// /// Create an open folder dialog box @@ -229,9 +240,9 @@ namespace MPF.UI.ViewModels private System.Windows.Controls.TextBox TextBoxForPathSetting(string name) => Parent.FindName(name + "TextBox") as System.Windows.Controls.TextBox; - #endregion +#endregion - #region Event Handlers +#region Event Handlers /// /// Handler for generic Click event @@ -254,9 +265,14 @@ namespace MPF.UI.ViewModels /// /// Test Redump credentials for validity /// +#if NET48 || NETSTANDARD2_1 private void OnRedumpTestClick(object sender, EventArgs e) => TestRedumpLogin(); +#else + private async void OnRedumpTestClick(object sender, EventArgs e) => + _ = await TestRedumpLogin(); +#endif - #endregion +#endregion } } diff --git a/RedumpLib/Data/Enumerations.cs b/RedumpLib/Data/Enumerations.cs index a260d1a5..94567acf 100644 --- a/RedumpLib/Data/Enumerations.cs +++ b/RedumpLib/Data/Enumerations.cs @@ -79,10 +79,10 @@ namespace RedumpLib.Data [HumanReadable(LongName = "DVD-9")] DVD9, - + [HumanReadable(LongName = "GD-ROM")] GDROM, - + [HumanReadable(LongName = "HD-DVD SL")] HDDVDSL, @@ -91,22 +91,22 @@ namespace RedumpLib.Data [HumanReadable(LongName = "MIL-CD")] MILCD, - + [HumanReadable(LongName = "Nintendo GameCube Game Disc")] NintendoGameCubeGameDisc, - + [HumanReadable(LongName = "Nintendo Wii Optical Disc SL")] NintendoWiiOpticalDiscSL, [HumanReadable(LongName = "Nintendo Wii Optical Disc DL")] NintendoWiiOpticalDiscDL, - + [HumanReadable(LongName = "Nintendo Wii U Optical Disc SL")] NintendoWiiUOpticalDiscSL, - + [HumanReadable(LongName = "UMD SL")] UMDSL, - + [HumanReadable(LongName = "UMD DL")] UMDDL, } @@ -492,7 +492,7 @@ namespace RedumpLib.Data [Language(LongName = "Erzya", ThreeLetterCode = "myv")] Erzya, - [Language(LongName = "Esperanto", TwoLetterCode="eo", ThreeLetterCode = "epo")] + [Language(LongName = "Esperanto", TwoLetterCode = "eo", ThreeLetterCode = "epo")] Esperanto, [Language(LongName = "Estonian", TwoLetterCode = "et", ThreeLetterCode = "est")] @@ -1113,7 +1113,7 @@ namespace RedumpLib.Data // Ossetian; Ossetic [Language(LongName = "Ossetian", TwoLetterCode = "os", ThreeLetterCode = "oss")] Ossetian, - + #endregion #region P @@ -1774,7 +1774,7 @@ namespace RedumpLib.Data [HumanReadable(LongName = "Language selector")] LanguageSelector, - + [HumanReadable(LongName = "Options menu")] OptionsMenu, } @@ -2224,7 +2224,7 @@ namespace RedumpLib.Data [System(Category = SystemCategory.Computer, LongName = "IBM PC compatible", ShortName = "pc", HasCues = true, HasDat = true, HasLsd = true, HasSbi = true)] IBMPCcompatible, - + [System(Category = SystemCategory.Computer, LongName = "NEC PC-88 series", ShortName = "pc-88", HasCues = true, HasDat = true)] NECPC88series, @@ -2396,7 +2396,7 @@ namespace RedumpLib.Data [System(Category = SystemCategory.Arcade, LongName = "TAB-Austria Quizard", ShortName = "quizard", HasCues = true, HasDat = true)] TABAustriaQuizard, - + [System(Category = SystemCategory.Arcade, Available = false, LongName = "Tsunami TsuMo Multi-Game Motion System")] TsunamiTsuMoMultiGameMotionSystem, diff --git a/RedumpLib/Data/Extensions.cs b/RedumpLib/Data/Extensions.cs index cfd1eb41..0251bbfb 100644 --- a/RedumpLib/Data/Extensions.cs +++ b/RedumpLib/Data/Extensions.cs @@ -12,7 +12,7 @@ namespace RedumpLib.Data { #region Cross-Enumeration - /// + /// /// Get a list of valid MediaTypes for a given RedumpSystem /// /// RedumpSystem value to check @@ -2072,7 +2072,7 @@ namespace RedumpLib.Data } #endregion - + #region System Category /// diff --git a/RedumpLib/Data/SubmissionInfo.cs b/RedumpLib/Data/SubmissionInfo.cs index 9fb96403..d21b2d1c 100644 --- a/RedumpLib/Data/SubmissionInfo.cs +++ b/RedumpLib/Data/SubmissionInfo.cs @@ -67,7 +67,7 @@ namespace RedumpLib.Data [JsonProperty(PropertyName = "artifacts", DefaultValueHandling = DefaultValueHandling.Ignore)] public Dictionary Artifacts { get; set; } = new Dictionary(); - + public object Clone() { return new SubmissionInfo diff --git a/RedumpLib/RedumpLib.csproj b/RedumpLib/RedumpLib.csproj index 97d9f7ed..60ca090a 100644 --- a/RedumpLib/RedumpLib.csproj +++ b/RedumpLib/RedumpLib.csproj @@ -1,8 +1,7 @@ - + - net48;net6.0 - x86 + net48;netstandard2.1;net6.0 false diff --git a/RedumpLib/Web/RedumpHttpClient.cs b/RedumpLib/Web/RedumpHttpClient.cs new file mode 100644 index 00000000..4ab9cb2a --- /dev/null +++ b/RedumpLib/Web/RedumpHttpClient.cs @@ -0,0 +1,808 @@ +#if NET6_0 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using RedumpLib.Data; + +namespace RedumpLib.Web +{ + public class RedumpHttpClient : HttpClient + { + #region Properties + + /// + /// Determines if user is logged into Redump + /// + public bool LoggedIn { get; private set; } = false; + + /// + /// Determines if the user is a staff member + /// + public bool IsStaff { get; private set; } = false; + + #endregion + + /// + /// Constructor + /// + public RedumpHttpClient() + : base(new HttpClientHandler { UseCookies = true }) + { + } + + #region Credentials + + /// + /// Validate supplied credentials + /// + public async static Task<(bool?, string)> ValidateCredentials(string username, string password) + { + // If options are invalid or we're missing something key, just return + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) + return (false, null); + + // Try logging in with the supplied credentials otherwise + using RedumpHttpClient httpClient = new(); + + bool? loggedIn = await httpClient.Login(username, password); + if (loggedIn == true) + return (true, "Redump username and password accepted!"); + else if (loggedIn == false) + return (false, "Redump username and password denied!"); + else + return (null, "An error occurred validating your credentials!"); + } + + /// + /// Login to Redump, if possible + /// + /// Redump username + /// Redump password + /// True if the user could be logged in, false otherwise, null on error + public async Task Login(string username, string password) + { + // Credentials verification + if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password)) + { + Console.WriteLine("Credentials entered, will attempt Redump login..."); + } + else if (!string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password)) + { + Console.WriteLine("Only a username was specified, will not attempt Redump login..."); + return false; + } + else if (string.IsNullOrWhiteSpace(username)) + { + Console.WriteLine("No credentials entered, will not attempt Redump login..."); + return false; + } + + // HTTP encode the password + password = WebUtility.UrlEncode(password); + + // Attempt to login up to 3 times + for (int i = 0; i < 3; i++) + { + try + { + // Get the current token from the login page + var loginPage = await GetStringAsync(Constants.LoginUrl); + string token = Constants.TokenRegex.Match(loginPage).Groups[1].Value; + + // Construct the login request + var postContent = new StringContent($"form_sent=1&redirect_url=&csrf_token={token}&req_username={username}&req_password={password}&save_pass=0", Encoding.UTF8); + postContent.Headers.ContentType = MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded"); + + // Send the login request and get the result + var response = await PostAsync(Constants.LoginUrl, postContent); + string responseContent = await response?.Content?.ReadAsStringAsync(); + + if (string.IsNullOrWhiteSpace(responseContent)) + { + Console.WriteLine($"An error occurred while trying to log in on attempt {i}: No response"); + continue; + } + + if (responseContent.Contains("Incorrect username and/or password.")) + { + Console.WriteLine("Invalid credentials entered, continuing without logging in..."); + return false; + } + + // The user was able to be logged in + Console.WriteLine("Credentials accepted! Logged into Redump..."); + LoggedIn = true; + + // If the user is a moderator or staff, set accordingly + if (responseContent.Contains("http://forum.redump.org/forum/9/staff/")) + IsStaff = true; + + return true; + } + catch (Exception ex) + { + Console.WriteLine($"An exception occurred while trying to log in on attempt {i}: {ex}"); + } + } + + Console.WriteLine("Could not login to Redump in 3 attempts, continuing without logging in..."); + return false; + } + + #endregion + + #region Single Page Helpers + + /// + /// Process a Redump site page as a list of possible IDs or disc page + /// + /// Base URL to download using + /// List of IDs from the page, empty on error + public async Task> CheckSingleSitePage(string url) + { + List ids = new(); + + // Try up to 3 times to retrieve the data + string dumpsPage = await DownloadString(url, retries: 3); + + // If we have no dumps left + if (dumpsPage == null || dumpsPage.Contains("No discs found.")) + return ids; + + // If we have a single disc page already + if (dumpsPage.Contains("Download:")) + { + var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; + if (int.TryParse(value, out int id)) + ids.Add(id); + + return ids; + } + + // Otherwise, traverse each dump on the page + var matches = Constants.DiscRegex.Matches(dumpsPage); + foreach (Match match in matches) + { + try + { + if (int.TryParse(match.Groups[1].Value, out int value)) + ids.Add(value); + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + continue; + } + } + + return ids; + } + + /// + /// Process a Redump site page as a list of possible IDs or disc page + /// + /// Base URL to download using + /// Output directory to save data to + /// True to return on first error, false otherwise + /// True if the page could be downloaded, false otherwise + public async Task CheckSingleSitePage(string url, string outDir, bool failOnSingle) + { + // Try up to 3 times to retrieve the data + string dumpsPage = await DownloadString(url, retries: 3); + + // If we have no dumps left + if (dumpsPage == null || dumpsPage.Contains("No discs found.")) + return false; + + // If we have a single disc page already + if (dumpsPage.Contains("Download:")) + { + var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value; + if (int.TryParse(value, out int id)) + { + bool downloaded = await DownloadSingleSiteID(id, outDir, false); + if (!downloaded && failOnSingle) + return false; + } + + return false; + } + + // Otherwise, traverse each dump on the page + var matches = Constants.DiscRegex.Matches(dumpsPage); + foreach (Match match in matches) + { + try + { + if (int.TryParse(match.Groups[1].Value, out int value)) + { + bool downloaded = await DownloadSingleSiteID(value, outDir, false); + if (!downloaded && failOnSingle) + return false; + } + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + continue; + } + } + + return true; + } + + /// + /// Process a Redump WIP page as a list of possible IDs or disc page + /// + /// RedumpWebClient to access the packs + /// List of IDs from the page, empty on error + public async Task> CheckSingleWIPPage(string url) + { + List ids = new(); + + // Try up to 3 times to retrieve the data + string dumpsPage = await DownloadString(url, retries: 3); + + // If we have no dumps left + if (dumpsPage == null || dumpsPage.Contains("No discs found.")) + return ids; + + // Otherwise, traverse each dump on the page + var matches = Constants.NewDiscRegex.Matches(dumpsPage); + foreach (Match match in matches) + { + try + { + if (int.TryParse(match.Groups[2].Value, out int value)) + ids.Add(value); + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + continue; + } + } + + return ids; + } + + /// + /// Process a Redump WIP page as a list of possible IDs or disc page + /// + /// RedumpWebClient to access the packs + /// Output directory to save data to + /// True to return on first error, false otherwise + /// True if the page could be downloaded, false otherwise + public async Task CheckSingleWIPPage(string url, string outDir, bool failOnSingle) + { + // Try up to 3 times to retrieve the data + string dumpsPage = await DownloadString(url, retries: 3); + + // If we have no dumps left + if (dumpsPage == null || dumpsPage.Contains("No discs found.")) + return false; + + // Otherwise, traverse each dump on the page + var matches = Constants.NewDiscRegex.Matches(dumpsPage); + foreach (Match match in matches) + { + try + { + if (int.TryParse(match.Groups[2].Value, out int value)) + { + bool downloaded = await DownloadSingleWIPID(value, outDir, false); + if (!downloaded && failOnSingle) + return false; + } + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + continue; + } + } + + return true; + } + + #endregion + + #region Download Helpers + + /// + /// Download a single pack + /// + /// Base URL to download using + /// System to download packs for + /// Byte array containing the downloaded pack, null on error + public async Task DownloadSinglePack(string url, RedumpSystem? system) + { + try + { + return await GetByteArrayAsync(string.Format(url, system.ShortName())); + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + return null; + } + } + + /// + /// Download a single pack + /// + /// Base URL to download using + /// System to download packs for + /// Output directory to save data to + /// Named subfolder for the pack, used optionally + public async Task DownloadSinglePack(string url, RedumpSystem? system, string outDir, string subfolder) + { + try + { + // If no output directory is defined, use the current directory instead + if (string.IsNullOrWhiteSpace(outDir)) + outDir = Environment.CurrentDirectory; + + string tempfile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + string packUri = string.Format(url, system.ShortName()); + + // Make the call to get the pack + string remoteFileName = await DownloadFile(packUri, tempfile); + MoveOrDelete(tempfile, remoteFileName, outDir, subfolder); + return true; + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + return false; + } + } + + /// + /// Download an individual site ID data, if possible + /// + /// Redump disc ID to retrieve + /// String containing the page contents if successful, null on error + public async Task DownloadSingleSiteID(int id) + { + string paddedId = id.ToString().PadLeft(5, '0'); + Console.WriteLine($"Processing ID: {paddedId}"); + try + { + // Try up to 3 times to retrieve the data + string discPageUri = string.Format(Constants.DiscPageUrl, +id); + string discPage = await DownloadString(discPageUri, retries: 3); + + if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) + { + Console.WriteLine($"ID {paddedId} could not be found!"); + return null; + } + + Console.WriteLine($"ID {paddedId} has been successfully downloaded"); + return discPage; + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + return null; + } + } + + /// + /// Download an individual site ID data, if possible + /// + /// Redump disc ID to retrieve + /// Output directory to save data to + /// True to rename deleted entries, false otherwise + /// True if all data was downloaded, false otherwise + public async Task DownloadSingleSiteID(int id, string outDir, bool rename) + { + // If no output directory is defined, use the current directory instead + if (string.IsNullOrWhiteSpace(outDir)) + outDir = Environment.CurrentDirectory; + + string paddedId = id.ToString().PadLeft(5, '0'); + string paddedIdDir = Path.Combine(outDir, paddedId); + Console.WriteLine($"Processing ID: {paddedId}"); + try + { + // Try up to 3 times to retrieve the data + string discPageUri = string.Format(Constants.DiscPageUrl, +id); + string discPage = await DownloadString(discPageUri, retries: 3); + + if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist")) + { + try + { + if (rename) + { + if (Directory.Exists(paddedIdDir) && rename) + Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); + else + Directory.CreateDirectory(paddedIdDir + "-deleted"); + } + } + catch { } + + Console.WriteLine($"ID {paddedId} could not be found!"); + return false; + } + + // Check if the page has been updated since the last time it was downloaded, if possible + if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) + { + // Read in the cached file + var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); + + // Check for the last modified date in both pages + var oldResult = Constants.LastModifiedRegex.Match(oldDiscPage); + var newResult = Constants.LastModifiedRegex.Match(discPage); + + // If both pages contain the same modified date, skip it + if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) + { + Console.WriteLine($"ID {paddedId} has not been changed since last download"); + return false; + } + + // If neither page contains a modified date, skip it + else if (!oldResult.Success && !newResult.Success) + { + Console.WriteLine($"ID {paddedId} has not been changed since last download"); + return false; + } + } + + // Create ID subdirectory + Directory.CreateDirectory(paddedIdDir); + + // View Edit History + if (discPage.Contains($" + /// Download an individual WIP ID data, if possible + /// + /// Redump WIP disc ID to retrieve + /// String containing the page contents if successful, null on error + public async Task DownloadSingleWIPID(int id) + { + string paddedId = id.ToString().PadLeft(5, '0'); + Console.WriteLine($"Processing ID: {paddedId}"); + try + { + // Try up to 3 times to retrieve the data + string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); + string discPage = await DownloadString(discPageUri, retries: 3); + + if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) + { + Console.WriteLine($"ID {paddedId} could not be found!"); + return null; + } + + Console.WriteLine($"ID {paddedId} has been successfully downloaded"); + return discPage; + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + return null; + } + } + + /// + /// Download an individual WIP ID data, if possible + /// + /// Redump WIP disc ID to retrieve + /// Output directory to save data to + /// True to rename deleted entries, false otherwise + /// True if all data was downloaded, false otherwise + public async Task DownloadSingleWIPID(int id, string outDir, bool rename) + { + // If no output directory is defined, use the current directory instead + if (string.IsNullOrWhiteSpace(outDir)) + outDir = Environment.CurrentDirectory; + + string paddedId = id.ToString().PadLeft(5, '0'); + string paddedIdDir = Path.Combine(outDir, paddedId); + Console.WriteLine($"Processing ID: {paddedId}"); + try + { + // Try up to 3 times to retrieve the data + string discPageUri = string.Format(Constants.WipDiscPageUrl, +id); + string discPage = await DownloadString(discPageUri, retries: 3); + + if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist")) + { + try + { + if (rename) + { + if (Directory.Exists(paddedIdDir) && rename) + Directory.Move(paddedIdDir, paddedIdDir + "-deleted"); + else + Directory.CreateDirectory(paddedIdDir + "-deleted"); + } + } + catch { } + + Console.WriteLine($"ID {paddedId} could not be found!"); + return false; + } + + // Check if the page has been updated since the last time it was downloaded, if possible + if (File.Exists(Path.Combine(paddedIdDir, "disc.html"))) + { + // Read in the cached file + var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html")); + + // Check for the full match ID in both pages + var oldResult = Constants.FullMatchRegex.Match(oldDiscPage); + var newResult = Constants.FullMatchRegex.Match(discPage); + + // If both pages contain the same ID, skip it + if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value) + { + Console.WriteLine($"ID {paddedId} has not been changed since last download"); + return false; + } + + // If neither page contains an ID, skip it + else if (!oldResult.Success && !newResult.Success) + { + Console.WriteLine($"ID {paddedId} has not been changed since last download"); + return false; + } + } + + // Create ID subdirectory + Directory.CreateDirectory(paddedIdDir); + + // HTML + using (var discStreamWriter = File.CreateText(Path.Combine(paddedIdDir, "disc.html"))) + { + discStreamWriter.Write(discPage); + } + + Console.WriteLine($"ID {paddedId} has been successfully downloaded"); + return true; + } + catch (Exception ex) + { + Console.WriteLine($"An exception has occurred: {ex}"); + return false; + } + } + + #endregion + + #region Helpers + + /// + /// Download a set of packs + /// + /// Base URL to download using + /// Systems to download packs for + /// Name of the pack that is downloading + public async Task> DownloadPacks(string url, RedumpSystem?[] systems, string title) + { + var packsDictionary = new Dictionary(); + + Console.WriteLine($"Downloading {title}"); + foreach (var system in systems) + { + // If the system is invalid, we can't do anything + if (system == null || !system.IsAvailable()) + continue; + + // If we didn't have credentials + if (!LoggedIn && system.IsBanned()) + continue; + + // If the system is unknown, we can't do anything + string longName = system.LongName(); + if (string.IsNullOrWhiteSpace(longName)) + continue; + + Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}"); + byte[] pack = await DownloadSinglePack(url, system); + if (pack != null) + packsDictionary.Add(system.Value, pack); + } + + Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); + Console.WriteLine(); + + return packsDictionary; + } + + /// + /// Download a set of packs + /// + /// Base URL to download using + /// Systems to download packs for + /// Name of the pack that is downloading + /// Output directory to save data to + /// Named subfolder for the pack, used optionally + public async Task DownloadPacks(string url, RedumpSystem?[] systems, string title, string outDir, string subfolder) + { + Console.WriteLine($"Downloading {title}"); + foreach (var system in systems) + { + // If the system is invalid, we can't do anything + if (system == null || !system.IsAvailable()) + continue; + + // If we didn't have credentials + if (!LoggedIn && system.IsBanned()) + continue; + + // If the system is unknown, we can't do anything + string longName = system.LongName(); + if (string.IsNullOrWhiteSpace(longName)) + continue; + + Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}"); + await DownloadSinglePack(url, system, outDir, subfolder); + } + + Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}"); + Console.WriteLine(); + return true; + } + + /// + /// Download from a URI to a local file + /// + /// Remote URI to retrieve + /// Filename to write to + /// The remote filename from the URI, null on error + private async Task DownloadFile(string uri, string fileName) + { + // Make the call to get the file + var response = await GetAsync(uri); + if (response?.Content?.Headers == null || !response.IsSuccessStatusCode) + { + Console.WriteLine($"Could not download {uri}"); + return null; + } + + // Copy the data to a local temp file + using (var responseStream = await response.Content.ReadAsStreamAsync()) + using (var tempFileStream = File.OpenWrite(fileName)) + { + responseStream.CopyTo(tempFileStream); + } + + return response.Content.Headers.ContentDisposition?.FileName?.Replace("\"", ""); + } + + /// + /// Download from a URI to a string + /// + /// Remote URI to retrieve + /// Number of times to retry on error + /// String from the URI, null on error + private async Task DownloadString(string uri, int retries = 3) + { + // Only retry a positive number of times + if (retries <= 0) + return null; + + for (int i = 0; i < retries; i++) + { + try + { + return await GetStringAsync(uri); + } + catch { } + } + + return null; + } + + /// + /// Move a tempfile to a new name unless it aleady exists, in which case, delete the tempfile + /// + /// Path to existing temporary file + /// Path to new output file + /// Output directory to save data to + /// Optional subfolder to append to the path + private static void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder) + { + // If we don't have a file to move to, just delete the temp file + if (string.IsNullOrWhiteSpace(newfile)) + { + File.Delete(tempfile); + return; + } + + // If we have a subfolder, create it and update the newfile name + if (!string.IsNullOrWhiteSpace(subfolder)) + { + if (!Directory.Exists(Path.Combine(outDir, subfolder))) + Directory.CreateDirectory(Path.Combine(outDir, subfolder)); + + newfile = Path.Combine(subfolder, newfile); + } + + // If the file already exists, don't overwrite it + if (File.Exists(Path.Combine(outDir, newfile))) + File.Delete(tempfile); + else + File.Move(tempfile, Path.Combine(outDir, newfile)); + } + + #endregion + } +} + +#endif \ No newline at end of file diff --git a/RedumpLib/Web/RedumpWebClient.cs b/RedumpLib/Web/RedumpWebClient.cs index b329a9ec..807099e1 100644 --- a/RedumpLib/Web/RedumpWebClient.cs +++ b/RedumpLib/Web/RedumpWebClient.cs @@ -1,4 +1,6 @@ -using System; +#if NET48 || NETSTANDARD2_1 + +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -32,25 +34,26 @@ namespace RedumpLib.Web // If the response headers are null or empty if (ResponseHeaders == null || ResponseHeaders.Count == 0) return null; - + // If we don't have the response header we care about string headerValue = ResponseHeaders.Get("Content-Disposition"); if (string.IsNullOrWhiteSpace(headerValue)) return null; // Extract the filename from the value +#if NETSTANDARD2_1 + return headerValue[(headerValue.IndexOf("filename=") + 9)..].Replace("\"", ""); +#else return headerValue.Substring(headerValue.IndexOf("filename=") + 9).Replace("\"", ""); +#endif } /// protected override WebRequest GetWebRequest(Uri address) { WebRequest request = base.GetWebRequest(address); - HttpWebRequest webRequest = request as HttpWebRequest; - if (webRequest != null) - { + if (request is HttpWebRequest webRequest) webRequest.CookieContainer = m_container; - } return request; } @@ -65,8 +68,12 @@ namespace RedumpLib.Web return (false, null); // Try logging in with the supplied credentials otherwise +#if NETSTANDARD2_1 + using RedumpWebClient wc = new RedumpWebClient(); +#else using (RedumpWebClient wc = new RedumpWebClient()) { +#endif bool? loggedIn = wc.Login(username, password); if (loggedIn == true) return (true, "Redump username and password accepted!"); @@ -74,7 +81,9 @@ namespace RedumpLib.Web return (false, "Redump username and password denied!"); else return (null, "An error occurred validating your credentials!"); +#if NET48 } +#endif } /// @@ -524,7 +533,7 @@ namespace RedumpLib.Web // View Edit History if (discPage.Contains($"Path to new output file /// Output directory to save data to /// Optional subfolder to append to the path - private void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder) + private static void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder) { if (!string.IsNullOrWhiteSpace(newfile)) { @@ -824,3 +833,5 @@ namespace RedumpLib.Web #endregion } } + +#endif \ No newline at end of file