SabreTools/MPF
1
0
Fork 0
mirror of https://github.com/SabreTools/MPF.git synced 2026-07-02 17:24:48 +00:00
Code Issues 392 Packages Projects Releases 20 Wiki Activity

Give .NET 6 priority for web calls

Browse Source
This commit is contained in:
Matt Nadareski
2022-07-26 13:47:19 -07:00
parent d6be0a4154
commit 9a92dbdedb
10 changed files with 953 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELIST.md
View File

@@ -99,6 +99,7 @@
- Fix .bin file paths; update internal filename generation
- Disable nonstandard BD-ROM sizes
- Trim leading file paths for XBONE
- Give .NET 6 priority for web calls
### 2.3 (2022-02-05)
- Start overhauling Redump information pulling, again

4
MPF.Check/Program.cs
View File

@@ -33,7 +33,11 @@ namespace MPF.Check
protectionProgress.ProgressChanged += ProgressUpdated;
// Validate the supplied credentials
#if NET48 || NETSTANDARD2_1
(bool? _, string message) = RedumpWebClient.ValidateCredentials(options?.RedumpUsername, options?.RedumpPassword);
#else
(bool? _, string message) = RedumpHttpClient.ValidateCredentials(options?.RedumpUsername, options?.RedumpPassword).ConfigureAwait(false).GetAwaiter().GetResult();
#endif
if (!string.IsNullOrWhiteSpace(message))
Console.WriteLine(message);

95
MPF.Library/InfoTool.cs
View File

@@ -97,7 +97,11 @@ namespace MPF.Library
// Get a list of matching IDs for each line in the DAT
if (!string.IsNullOrEmpty(info.TracksAndWriteOffsets.ClrMameProData) && options.HasRedumpLogin)
#if NET48 || NETSTANDARD2_1
FillFromRedump(options, info, resultProgress);
#else
_ = await FillFromRedump(options, info, resultProgress);
#endif
// If we have both ClrMamePro and Size and Checksums data, remove the ClrMamePro
if (!string.IsNullOrWhiteSpace(info.SizeAndChecksums.CRC32))
@@ -561,9 +565,9 @@ namespace MPF.Library
}
}
#endregion
#endregion
#region Information Output
#region Information Output
/// <summary>
/// Compress log files to save space
@@ -1134,9 +1138,9 @@ namespace MPF.Library
AddIfExists(output, key, string.Join(", ", value.Select(o => o.ToString())), indent);
}
#endregion
#endregion
#region Normalization
#region Normalization
/// <summary>
/// Adjust the disc type based on size and layerbreak information
@@ -1257,9 +1261,9 @@ namespace MPF.Library
return (directory, filename);
}
#endregion
#endregion
#region Web Calls
#region Web Calls
/// <summary>
/// Create a new SubmissionInfo object from a disc page
@@ -1444,11 +1448,19 @@ namespace MPF.Library
/// <param name="wc">RedumpWebClient for making the connection</param>
/// <param name="info">Existing SubmissionInfo object to fill</param>
/// <param name="id">Redump disc ID to retrieve</param>
private static void FillFromId(RedumpWebClient wc, SubmissionInfo info, int id)
#if NET48 || NETSTANDARD2_1
private static bool FillFromId(RedumpWebClient wc, SubmissionInfo info, int id)
{
string discData = wc.DownloadSingleSiteID(id);
if (string.IsNullOrEmpty(discData))
return;
return false;
#else
private async static Task<bool> FillFromId(RedumpHttpClient wc, SubmissionInfo info, int id)
{
string discData = await wc.DownloadSingleSiteID(id);
if (string.IsNullOrEmpty(discData))
return false;
#endif
// Title, Disc Number/Letter, Disc Title
var match = Constants.TitleRegex.Match(discData);
@@ -1736,6 +1748,8 @@ namespace MPF.Library
else
info.LastModified = null;
}
return true;
}
/// <summary>
@@ -1744,25 +1758,37 @@ namespace MPF.Library
/// <param name="options">Options object representing user-defined options</param>
/// <param name="info">Existing SubmissionInfo object to fill</param>
/// <param name="resultProgress">Optional result progress callback</param>
private static void FillFromRedump(Options options, SubmissionInfo info, IProgress<Result> resultProgress = null)
#if NET48 || NETSTANDARD2_1
private static bool FillFromRedump(Options options, SubmissionInfo info, IProgress<Result> resultProgress = null)
#else
private async static Task<bool> FillFromRedump(Options options, SubmissionInfo info, IProgress<Result> resultProgress = null)
#endif
{
// Set the current dumper based on username
info.DumpersAndStatus.Dumpers = new string[] { options.RedumpUsername };
info.PartiallyMatchedIDs = new List<int>();
#if NET48 || NETSTANDARD2_1
using (RedumpWebClient wc = new RedumpWebClient())
#else
using (RedumpHttpClient wc = new RedumpHttpClient())
#endif
{
// Login to Redump
#if NET48 || NETSTANDARD2_1
bool? loggedIn = wc.Login(options.RedumpUsername, options.RedumpPassword);
#else
bool? loggedIn = await wc.Login(options.RedumpUsername, options.RedumpPassword);
#endif
if (loggedIn == null)
{
resultProgress?.Report(Result.Failure("There was an unknown error connecting to Redump"));
return;
return false;
}
else if (loggedIn == false)
{
// Don't log the as a failure or error
return;
return false;
}
// Setup the full-track checks
@@ -1774,7 +1800,11 @@ namespace MPF.Library
string[] splitData = info.TracksAndWriteOffsets.ClrMameProData.Split('\n');
foreach (string hashData in splitData)
{
#if NET48 || NETSTANDARD2_1
(bool singleFound, List<int> foundIds) = ValidateSingleTrack(wc, info, hashData, resultProgress);
#else
(bool singleFound, List<int> foundIds) = await ValidateSingleTrack(wc, info, hashData, resultProgress);
#endif
// Ensure that all tracks are found
allFound &= singleFound;
@@ -1806,19 +1836,28 @@ namespace MPF.Library
// Exit early if one failed or there are no matched IDs
if (!allFound || fullyMatchedIDs.Count == 0)
return;
return false;
// Find the first matched ID where the track count matches, we can grab a bunch of info from it
int totalMatchedIDsCount = fullyMatchedIDs.Count;
for (int i = 0; i < totalMatchedIDsCount; i++)
{
// Skip if the track count doesn't match
#if NET48 || NETSTANDARD2_1
if (!ValidateTrackCount(wc, fullyMatchedIDs[i], splitData.Length))
continue;
#else
if (!await ValidateTrackCount(wc, fullyMatchedIDs[i], splitData.Length))
continue;
#endif
// Fill in the fields from the existing ID
resultProgress?.Report(Result.Success($"Filling fields from existing ID {fullyMatchedIDs[i]}..."));
#if NET48 || NETSTANDARD2_1
FillFromId(wc, info, fullyMatchedIDs[i]);
#else
_ = await FillFromId(wc, info, fullyMatchedIDs[i]);
#endif
resultProgress?.Report(Result.Success("Information filling complete!"));
// Set the fully matched ID to the current
@@ -1835,6 +1874,8 @@ namespace MPF.Library
info.PartiallyMatchedIDs.Remove(info.FullyMatchedID.Value);
}
}
return true;
}
/// <summary>
@@ -1873,7 +1914,11 @@ namespace MPF.Library
/// <param name="wc">RedumpWebClient for making the connection</param>
/// <param name="query">Query string to attempt to search for</param>
/// <returns>All disc IDs for the given query, null on error</returns>
#if NET48 || NETSTANDARD2_1
private static List<int> ListSearchResults(RedumpWebClient wc, string query)
#else
private async static Task<List<int>> ListSearchResults(RedumpHttpClient wc, string query)
#endif
{
List<int> ids = new List<int>();
@@ -1894,7 +1939,11 @@ namespace MPF.Library
int pageNumber = 1;
while (true)
{
#if NET48 || NETSTANDARD2_1
List<int> pageIds = wc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++));
#else
List<int> pageIds = await wc.CheckSingleSitePage(string.Format(Constants.QuickSearchUrl, query, pageNumber++));
#endif
ids.AddRange(pageIds);
if (pageIds.Count <= 1)
break;
@@ -1917,7 +1966,11 @@ namespace MPF.Library
/// <param name="hashData">DAT-formatted hash data to parse out</param>
/// <param name="resultProgress">Optional result progress callback</param>
/// <returns>True if the track was found, false otherwise; List of found values, if possible</returns>
#if NET48 || NETSTANDARD2_1
private static (bool, List<int>) ValidateSingleTrack(RedumpWebClient wc, SubmissionInfo info, string hashData, IProgress<Result> resultProgress = null)
#else
private async static Task<(bool, List<int>)> ValidateSingleTrack(RedumpHttpClient wc, SubmissionInfo info, string hashData, IProgress<Result> resultProgress = null)
#endif
{
// If the line isn't parseable, we can't validate
if (!GetISOHashValues(hashData, out long _, out string _, out string _, out string sha1))
@@ -1927,7 +1980,11 @@ namespace MPF.Library
}
// Get all matching IDs for the track
#if NET48 || NETSTANDARD2_1
List<int> newIds = ListSearchResults(wc, sha1);
#else
List<int> newIds = await ListSearchResults(wc, sha1);
#endif
// If we got null back, there was an error
if (newIds == null)
@@ -1956,10 +2013,18 @@ namespace MPF.Library
/// <param name="id">Redump disc ID to retrieve</param>
/// <param name="localCount">Local count of tracks for the current disc</param>
/// <returns>True if the track count matches, false otherwise</returns>
#if NET48 || NETSTANDARD2_1
private static bool ValidateTrackCount(RedumpWebClient wc, int id, int localCount)
#else
private async static Task<bool> ValidateTrackCount(RedumpHttpClient wc, int id, int localCount)
#endif
{
// If we can't pull the remote data, we can't match
#if NET48 || NETSTANDARD2_1
string discData = wc.DownloadSingleSiteID(id);
#else
string discData = await wc.DownloadSingleSiteID(id);
#endif
if (string.IsNullOrEmpty(discData))
return false;
@@ -1978,9 +2043,9 @@ namespace MPF.Library
return localCount == remoteCount;
}
#endregion
#endregion
#region Helpers
#region Helpers
/// <summary>
/// Format a single site tag to string
@@ -2208,6 +2273,6 @@ namespace MPF.Library
return sorted;
}
#endregion
#endregion
}
}

28
MPF/ViewModels/OptionsViewModel.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Forms;
using MPF.Core.Data;
@@ -187,20 +188,30 @@ namespace MPF.UI.ViewModels
/// <summary>
/// Test Redump login credentials
/// </summary>
private void TestRedumpLogin()
#if NET48 || NETSTANDARD2_1
private bool? TestRedumpLogin()
#else
private async Task<bool?> TestRedumpLogin()
#endif
{
#if NET48 || NETSTANDARD2_1
(bool? success, string message) = RedumpWebClient.ValidateCredentials(Parent.RedumpUsernameTextBox.Text, Parent.RedumpPasswordBox.Password);
#else
(bool? success, string message) = await RedumpHttpClient.ValidateCredentials(Parent.RedumpUsernameTextBox.Text, Parent.RedumpPasswordBox.Password);
#endif
if (success == true)
CustomMessageBox.Show(Parent, message, "Success", MessageBoxButton.OK, MessageBoxImage.Information);
else if (success == false)
CustomMessageBox.Show(Parent, message, "Failure", MessageBoxButton.OK, MessageBoxImage.Error);
else
CustomMessageBox.Show(Parent, message, "Error", MessageBoxButton.OK, MessageBoxImage.Error);
return success;
}
#endregion
#endregion
#region UI Functionality
#region UI Functionality
/// <summary>
/// Create an open folder dialog box
@@ -229,9 +240,9 @@ namespace MPF.UI.ViewModels
private System.Windows.Controls.TextBox TextBoxForPathSetting(string name) =>
Parent.FindName(name + "TextBox") as System.Windows.Controls.TextBox;
#endregion
#endregion
#region Event Handlers
#region Event Handlers
/// <summary>
/// Handler for generic Click event
@@ -254,9 +265,14 @@ namespace MPF.UI.ViewModels
/// <summary>
/// Test Redump credentials for validity
/// </summary>
#if NET48 || NETSTANDARD2_1
private void OnRedumpTestClick(object sender, EventArgs e) =>
TestRedumpLogin();
#else
private async void OnRedumpTestClick(object sender, EventArgs e) =>
_ = await TestRedumpLogin();
#endif
#endregion
#endregion
}
}

24
RedumpLib/Data/Enumerations.cs
View File

@@ -79,10 +79,10 @@ namespace RedumpLib.Data
[HumanReadable(LongName = "DVD-9")]
DVD9,
[HumanReadable(LongName = "GD-ROM")]
GDROM,
[HumanReadable(LongName = "HD-DVD SL")]
HDDVDSL,
@@ -91,22 +91,22 @@ namespace RedumpLib.Data
[HumanReadable(LongName = "MIL-CD")]
MILCD,
[HumanReadable(LongName = "Nintendo GameCube Game Disc")]
NintendoGameCubeGameDisc,
[HumanReadable(LongName = "Nintendo Wii Optical Disc SL")]
NintendoWiiOpticalDiscSL,
[HumanReadable(LongName = "Nintendo Wii Optical Disc DL")]
NintendoWiiOpticalDiscDL,
[HumanReadable(LongName = "Nintendo Wii U Optical Disc SL")]
NintendoWiiUOpticalDiscSL,
[HumanReadable(LongName = "UMD SL")]
UMDSL,
[HumanReadable(LongName = "UMD DL")]
UMDDL,
}
@@ -492,7 +492,7 @@ namespace RedumpLib.Data
[Language(LongName = "Erzya", ThreeLetterCode = "myv")]
Erzya,
[Language(LongName = "Esperanto", TwoLetterCode="eo", ThreeLetterCode = "epo")]
[Language(LongName = "Esperanto", TwoLetterCode = "eo", ThreeLetterCode = "epo")]
Esperanto,
[Language(LongName = "Estonian", TwoLetterCode = "et", ThreeLetterCode = "est")]
@@ -1113,7 +1113,7 @@ namespace RedumpLib.Data
// Ossetian; Ossetic
[Language(LongName = "Ossetian", TwoLetterCode = "os", ThreeLetterCode = "oss")]
Ossetian,
#endregion
#region P
@@ -1774,7 +1774,7 @@ namespace RedumpLib.Data
[HumanReadable(LongName = "Language selector")]
LanguageSelector,
[HumanReadable(LongName = "Options menu")]
OptionsMenu,
}
@@ -2224,7 +2224,7 @@ namespace RedumpLib.Data
[System(Category = SystemCategory.Computer, LongName = "IBM PC compatible", ShortName = "pc", HasCues = true, HasDat = true, HasLsd = true, HasSbi = true)]
IBMPCcompatible,
[System(Category = SystemCategory.Computer, LongName = "NEC PC-88 series", ShortName = "pc-88", HasCues = true, HasDat = true)]
NECPC88series,
@@ -2396,7 +2396,7 @@ namespace RedumpLib.Data
[System(Category = SystemCategory.Arcade, LongName = "TAB-Austria Quizard", ShortName = "quizard", HasCues = true, HasDat = true)]
TABAustriaQuizard,
[System(Category = SystemCategory.Arcade, Available = false, LongName = "Tsunami TsuMo Multi-Game Motion System")]
TsunamiTsuMoMultiGameMotionSystem,

4
RedumpLib/Data/Extensions.cs
View File

@@ -12,7 +12,7 @@ namespace RedumpLib.Data
{
#region Cross-Enumeration
/// <summary>
/// <summary>
/// Get a list of valid MediaTypes for a given RedumpSystem
/// </summary>
/// <param name="system">RedumpSystem value to check</param>
@@ -2072,7 +2072,7 @@ namespace RedumpLib.Data
}
#endregion
#region System Category
/// <summary>

2
RedumpLib/Data/SubmissionInfo.cs
View File

@@ -67,7 +67,7 @@ namespace RedumpLib.Data
[JsonProperty(PropertyName = "artifacts", DefaultValueHandling = DefaultValueHandling.Ignore)]
public Dictionary<string, string> Artifacts { get; set; } = new Dictionary<string, string>();
public object Clone()
{
return new SubmissionInfo

5
RedumpLib/RedumpLib.csproj
View File

@@ -1,8 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net48;net6.0</TargetFrameworks>
<PlatformTarget>x86</PlatformTarget>
<TargetFrameworks>net48;netstandard2.1;net6.0</TargetFrameworks>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
</PropertyGroup>

808
RedumpLib/Web/RedumpHttpClient.cs Normal file
View File

@@ -0,0 +1,808 @@
#if NET6_0
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using RedumpLib.Data;
namespace RedumpLib.Web
{
public class RedumpHttpClient : HttpClient
{
#region Properties
/// <summary>
/// Determines if user is logged into Redump
/// </summary>
public bool LoggedIn { get; private set; } = false;
/// <summary>
/// Determines if the user is a staff member
/// </summary>
public bool IsStaff { get; private set; } = false;
#endregion
/// <summary>
/// Constructor
/// </summary>
public RedumpHttpClient()
: base(new HttpClientHandler { UseCookies = true })
{
}
#region Credentials
/// <summary>
/// Validate supplied credentials
/// </summary>
public async static Task<(bool?, string)> ValidateCredentials(string username, string password)
{
// If options are invalid or we're missing something key, just return
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
return (false, null);
// Try logging in with the supplied credentials otherwise
using RedumpHttpClient httpClient = new();
bool? loggedIn = await httpClient.Login(username, password);
if (loggedIn == true)
return (true, "Redump username and password accepted!");
else if (loggedIn == false)
return (false, "Redump username and password denied!");
else
return (null, "An error occurred validating your credentials!");
}
/// <summary>
/// Login to Redump, if possible
/// </summary>
/// <param name="username">Redump username</param>
/// <param name="password">Redump password</param>
/// <returns>True if the user could be logged in, false otherwise, null on error</returns>
public async Task<bool?> Login(string username, string password)
{
// Credentials verification
if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password))
{
Console.WriteLine("Credentials entered, will attempt Redump login...");
}
else if (!string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password))
{
Console.WriteLine("Only a username was specified, will not attempt Redump login...");
return false;
}
else if (string.IsNullOrWhiteSpace(username))
{
Console.WriteLine("No credentials entered, will not attempt Redump login...");
return false;
}
// HTTP encode the password
password = WebUtility.UrlEncode(password);
// Attempt to login up to 3 times
for (int i = 0; i < 3; i++)
{
try
{
// Get the current token from the login page
var loginPage = await GetStringAsync(Constants.LoginUrl);
string token = Constants.TokenRegex.Match(loginPage).Groups[1].Value;
// Construct the login request
var postContent = new StringContent($"form_sent=1&redirect_url=&csrf_token={token}&req_username={username}&req_password={password}&save_pass=0", Encoding.UTF8);
postContent.Headers.ContentType = MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded");
// Send the login request and get the result
var response = await PostAsync(Constants.LoginUrl, postContent);
string responseContent = await response?.Content?.ReadAsStringAsync();
if (string.IsNullOrWhiteSpace(responseContent))
{
Console.WriteLine($"An error occurred while trying to log in on attempt {i}: No response");
continue;
}
if (responseContent.Contains("Incorrect username and/or password."))
{
Console.WriteLine("Invalid credentials entered, continuing without logging in...");
return false;
}
// The user was able to be logged in
Console.WriteLine("Credentials accepted! Logged into Redump...");
LoggedIn = true;
// If the user is a moderator or staff, set accordingly
if (responseContent.Contains("http://forum.redump.org/forum/9/staff/"))
IsStaff = true;
return true;
}
catch (Exception ex)
{
Console.WriteLine($"An exception occurred while trying to log in on attempt {i}: {ex}");
}
}
Console.WriteLine("Could not login to Redump in 3 attempts, continuing without logging in...");
return false;
}
#endregion
#region Single Page Helpers
/// <summary>
/// Process a Redump site page as a list of possible IDs or disc page
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <returns>List of IDs from the page, empty on error</returns>
public async Task<List<int>> CheckSingleSitePage(string url)
{
List<int> ids = new();
// Try up to 3 times to retrieve the data
string dumpsPage = await DownloadString(url, retries: 3);
// If we have no dumps left
if (dumpsPage == null || dumpsPage.Contains("No discs found."))
return ids;
// If we have a single disc page already
if (dumpsPage.Contains("<b>Download:</b>"))
{
var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value;
if (int.TryParse(value, out int id))
ids.Add(id);
return ids;
}
// Otherwise, traverse each dump on the page
var matches = Constants.DiscRegex.Matches(dumpsPage);
foreach (Match match in matches)
{
try
{
if (int.TryParse(match.Groups[1].Value, out int value))
ids.Add(value);
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
continue;
}
}
return ids;
}
/// <summary>
/// Process a Redump site page as a list of possible IDs or disc page
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="failOnSingle">True to return on first error, false otherwise</param>
/// <returns>True if the page could be downloaded, false otherwise</returns>
public async Task<bool> CheckSingleSitePage(string url, string outDir, bool failOnSingle)
{
// Try up to 3 times to retrieve the data
string dumpsPage = await DownloadString(url, retries: 3);
// If we have no dumps left
if (dumpsPage == null || dumpsPage.Contains("No discs found."))
return false;
// If we have a single disc page already
if (dumpsPage.Contains("<b>Download:</b>"))
{
var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value;
if (int.TryParse(value, out int id))
{
bool downloaded = await DownloadSingleSiteID(id, outDir, false);
if (!downloaded && failOnSingle)
return false;
}
return false;
}
// Otherwise, traverse each dump on the page
var matches = Constants.DiscRegex.Matches(dumpsPage);
foreach (Match match in matches)
{
try
{
if (int.TryParse(match.Groups[1].Value, out int value))
{
bool downloaded = await DownloadSingleSiteID(value, outDir, false);
if (!downloaded && failOnSingle)
return false;
}
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
continue;
}
}
return true;
}
/// <summary>
/// Process a Redump WIP page as a list of possible IDs or disc page
/// </summary>
/// <param name="wc">RedumpWebClient to access the packs</param>
/// <returns>List of IDs from the page, empty on error</returns>
public async Task<List<int>> CheckSingleWIPPage(string url)
{
List<int> ids = new();
// Try up to 3 times to retrieve the data
string dumpsPage = await DownloadString(url, retries: 3);
// If we have no dumps left
if (dumpsPage == null || dumpsPage.Contains("No discs found."))
return ids;
// Otherwise, traverse each dump on the page
var matches = Constants.NewDiscRegex.Matches(dumpsPage);
foreach (Match match in matches)
{
try
{
if (int.TryParse(match.Groups[2].Value, out int value))
ids.Add(value);
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
continue;
}
}
return ids;
}
/// <summary>
/// Process a Redump WIP page as a list of possible IDs or disc page
/// </summary>
/// <param name="wc">RedumpWebClient to access the packs</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="failOnSingle">True to return on first error, false otherwise</param>
/// <returns>True if the page could be downloaded, false otherwise</returns>
public async Task<bool> CheckSingleWIPPage(string url, string outDir, bool failOnSingle)
{
// Try up to 3 times to retrieve the data
string dumpsPage = await DownloadString(url, retries: 3);
// If we have no dumps left
if (dumpsPage == null || dumpsPage.Contains("No discs found."))
return false;
// Otherwise, traverse each dump on the page
var matches = Constants.NewDiscRegex.Matches(dumpsPage);
foreach (Match match in matches)
{
try
{
if (int.TryParse(match.Groups[2].Value, out int value))
{
bool downloaded = await DownloadSingleWIPID(value, outDir, false);
if (!downloaded && failOnSingle)
return false;
}
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
continue;
}
}
return true;
}
#endregion
#region Download Helpers
/// <summary>
/// Download a single pack
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <param name="system">System to download packs for</param>
/// <returns>Byte array containing the downloaded pack, null on error</returns>
public async Task<byte[]> DownloadSinglePack(string url, RedumpSystem? system)
{
try
{
return await GetByteArrayAsync(string.Format(url, system.ShortName()));
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return null;
}
}
/// <summary>
/// Download a single pack
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <param name="system">System to download packs for</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="subfolder">Named subfolder for the pack, used optionally</param>
public async Task<bool> DownloadSinglePack(string url, RedumpSystem? system, string outDir, string subfolder)
{
try
{
// If no output directory is defined, use the current directory instead
if (string.IsNullOrWhiteSpace(outDir))
outDir = Environment.CurrentDirectory;
string tempfile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString());
string packUri = string.Format(url, system.ShortName());
// Make the call to get the pack
string remoteFileName = await DownloadFile(packUri, tempfile);
MoveOrDelete(tempfile, remoteFileName, outDir, subfolder);
return true;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return false;
}
}
/// <summary>
/// Download an individual site ID data, if possible
/// </summary>
/// <param name="id">Redump disc ID to retrieve</param>
/// <returns>String containing the page contents if successful, null on error</returns>
public async Task<string> DownloadSingleSiteID(int id)
{
string paddedId = id.ToString().PadLeft(5, '0');
Console.WriteLine($"Processing ID: {paddedId}");
try
{
// Try up to 3 times to retrieve the data
string discPageUri = string.Format(Constants.DiscPageUrl, +id);
string discPage = await DownloadString(discPageUri, retries: 3);
if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist"))
{
Console.WriteLine($"ID {paddedId} could not be found!");
return null;
}
Console.WriteLine($"ID {paddedId} has been successfully downloaded");
return discPage;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return null;
}
}
/// <summary>
/// Download an individual site ID data, if possible
/// </summary>
/// <param name="id">Redump disc ID to retrieve</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="rename">True to rename deleted entries, false otherwise</param>
/// <returns>True if all data was downloaded, false otherwise</returns>
public async Task<bool> DownloadSingleSiteID(int id, string outDir, bool rename)
{
// If no output directory is defined, use the current directory instead
if (string.IsNullOrWhiteSpace(outDir))
outDir = Environment.CurrentDirectory;
string paddedId = id.ToString().PadLeft(5, '0');
string paddedIdDir = Path.Combine(outDir, paddedId);
Console.WriteLine($"Processing ID: {paddedId}");
try
{
// Try up to 3 times to retrieve the data
string discPageUri = string.Format(Constants.DiscPageUrl, +id);
string discPage = await DownloadString(discPageUri, retries: 3);
if (discPage == null || discPage.Contains($"Disc with ID \"{id}\" doesn't exist"))
{
try
{
if (rename)
{
if (Directory.Exists(paddedIdDir) && rename)
Directory.Move(paddedIdDir, paddedIdDir + "-deleted");
else
Directory.CreateDirectory(paddedIdDir + "-deleted");
}
}
catch { }
Console.WriteLine($"ID {paddedId} could not be found!");
return false;
}
// Check if the page has been updated since the last time it was downloaded, if possible
if (File.Exists(Path.Combine(paddedIdDir, "disc.html")))
{
// Read in the cached file
var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html"));
// Check for the last modified date in both pages
var oldResult = Constants.LastModifiedRegex.Match(oldDiscPage);
var newResult = Constants.LastModifiedRegex.Match(discPage);
// If both pages contain the same modified date, skip it
if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value)
{
Console.WriteLine($"ID {paddedId} has not been changed since last download");
return false;
}
// If neither page contains a modified date, skip it
else if (!oldResult.Success && !newResult.Success)
{
Console.WriteLine($"ID {paddedId} has not been changed since last download");
return false;
}
}
// Create ID subdirectory
Directory.CreateDirectory(paddedIdDir);
// View Edit History
if (discPage.Contains($"<a href=\"/disc/{id}/changes/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.ChangesExt, Path.Combine(paddedIdDir, "changes.html"));
// CUE
if (discPage.Contains($"<a href=\"/disc/{id}/cue/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.CueExt, Path.Combine(paddedIdDir, paddedId + ".cue"));
// Edit disc
if (discPage.Contains($"<a href=\"/disc/{id}/edit/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.EditExt, Path.Combine(paddedIdDir, "edit.html"));
// GDI
if (discPage.Contains($"<a href=\"/disc/{id}/gdi/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.GdiExt, Path.Combine(paddedIdDir, paddedId + ".gdi"));
// KEYS
if (discPage.Contains($"<a href=\"/disc/{id}/key/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.KeyExt, Path.Combine(paddedIdDir, paddedId + ".key"));
// LSD
if (discPage.Contains($"<a href=\"/disc/{id}/lsd/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.LsdExt, Path.Combine(paddedIdDir, paddedId + ".lsd"));
// MD5
if (discPage.Contains($"<a href=\"/disc/{id}/md5/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.Md5Ext, Path.Combine(paddedIdDir, paddedId + ".md5"));
// Review WIP entry
if (Constants.NewDiscRegex.IsMatch(discPage))
{
var match = Constants.NewDiscRegex.Match(discPage);
_ = await DownloadFile(string.Format(Constants.WipDiscPageUrl, match.Groups[2].Value), Path.Combine(paddedIdDir, "newdisc.html"));
}
// SBI
if (discPage.Contains($"<a href=\"/disc/{id}/sbi/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.SbiExt, Path.Combine(paddedIdDir, paddedId + ".sbi"));
// SFV
if (discPage.Contains($"<a href=\"/disc/{id}/sfv/\""))
await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.SfvExt, Path.Combine(paddedIdDir, paddedId + ".sfv"));
// SHA1
if (discPage.Contains($"<a href=\"/disc/{id}/sha1/\""))
_ = await DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.Sha1Ext, Path.Combine(paddedIdDir, paddedId + ".sha1"));
// HTML (Last in case of errors)
using (var discStreamWriter = File.CreateText(Path.Combine(paddedIdDir, "disc.html")))
{
discStreamWriter.Write(discPage);
}
Console.WriteLine($"ID {paddedId} has been successfully downloaded");
return true;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return false;
}
}
/// <summary>
/// Download an individual WIP ID data, if possible
/// </summary>
/// <param name="id">Redump WIP disc ID to retrieve</param>
/// <returns>String containing the page contents if successful, null on error</returns>
public async Task<string> DownloadSingleWIPID(int id)
{
string paddedId = id.ToString().PadLeft(5, '0');
Console.WriteLine($"Processing ID: {paddedId}");
try
{
// Try up to 3 times to retrieve the data
string discPageUri = string.Format(Constants.WipDiscPageUrl, +id);
string discPage = await DownloadString(discPageUri, retries: 3);
if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist"))
{
Console.WriteLine($"ID {paddedId} could not be found!");
return null;
}
Console.WriteLine($"ID {paddedId} has been successfully downloaded");
return discPage;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return null;
}
}
/// <summary>
/// Download an individual WIP ID data, if possible
/// </summary>
/// <param name="id">Redump WIP disc ID to retrieve</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="rename">True to rename deleted entries, false otherwise</param>
/// <returns>True if all data was downloaded, false otherwise</returns>
public async Task<bool> DownloadSingleWIPID(int id, string outDir, bool rename)
{
// If no output directory is defined, use the current directory instead
if (string.IsNullOrWhiteSpace(outDir))
outDir = Environment.CurrentDirectory;
string paddedId = id.ToString().PadLeft(5, '0');
string paddedIdDir = Path.Combine(outDir, paddedId);
Console.WriteLine($"Processing ID: {paddedId}");
try
{
// Try up to 3 times to retrieve the data
string discPageUri = string.Format(Constants.WipDiscPageUrl, +id);
string discPage = await DownloadString(discPageUri, retries: 3);
if (discPage == null || discPage.Contains($"WIP disc with ID \"{id}\" doesn't exist"))
{
try
{
if (rename)
{
if (Directory.Exists(paddedIdDir) && rename)
Directory.Move(paddedIdDir, paddedIdDir + "-deleted");
else
Directory.CreateDirectory(paddedIdDir + "-deleted");
}
}
catch { }
Console.WriteLine($"ID {paddedId} could not be found!");
return false;
}
// Check if the page has been updated since the last time it was downloaded, if possible
if (File.Exists(Path.Combine(paddedIdDir, "disc.html")))
{
// Read in the cached file
var oldDiscPage = File.ReadAllText(Path.Combine(paddedIdDir, "disc.html"));
// Check for the full match ID in both pages
var oldResult = Constants.FullMatchRegex.Match(oldDiscPage);
var newResult = Constants.FullMatchRegex.Match(discPage);
// If both pages contain the same ID, skip it
if (oldResult.Success && newResult.Success && oldResult.Groups[1].Value == newResult.Groups[1].Value)
{
Console.WriteLine($"ID {paddedId} has not been changed since last download");
return false;
}
// If neither page contains an ID, skip it
else if (!oldResult.Success && !newResult.Success)
{
Console.WriteLine($"ID {paddedId} has not been changed since last download");
return false;
}
}
// Create ID subdirectory
Directory.CreateDirectory(paddedIdDir);
// HTML
using (var discStreamWriter = File.CreateText(Path.Combine(paddedIdDir, "disc.html")))
{
discStreamWriter.Write(discPage);
}
Console.WriteLine($"ID {paddedId} has been successfully downloaded");
return true;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return false;
}
}
#endregion
#region Helpers
/// <summary>
/// Download a set of packs
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <param name="system">Systems to download packs for</param>
/// <param name="title">Name of the pack that is downloading</param>
public async Task<Dictionary<RedumpSystem, byte[]>> DownloadPacks(string url, RedumpSystem?[] systems, string title)
{
var packsDictionary = new Dictionary<RedumpSystem, byte[]>();
Console.WriteLine($"Downloading {title}");
foreach (var system in systems)
{
// If the system is invalid, we can't do anything
if (system == null || !system.IsAvailable())
continue;
// If we didn't have credentials
if (!LoggedIn && system.IsBanned())
continue;
// If the system is unknown, we can't do anything
string longName = system.LongName();
if (string.IsNullOrWhiteSpace(longName))
continue;
Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}");
byte[] pack = await DownloadSinglePack(url, system);
if (pack != null)
packsDictionary.Add(system.Value, pack);
}
Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}");
Console.WriteLine();
return packsDictionary;
}
/// <summary>
/// Download a set of packs
/// </summary>
/// <param name="url">Base URL to download using</param>
/// <param name="systems">Systems to download packs for</param>
/// <param name="title">Name of the pack that is downloading</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="subfolder">Named subfolder for the pack, used optionally</param>
public async Task<bool> DownloadPacks(string url, RedumpSystem?[] systems, string title, string outDir, string subfolder)
{
Console.WriteLine($"Downloading {title}");
foreach (var system in systems)
{
// If the system is invalid, we can't do anything
if (system == null || !system.IsAvailable())
continue;
// If we didn't have credentials
if (!LoggedIn && system.IsBanned())
continue;
// If the system is unknown, we can't do anything
string longName = system.LongName();
if (string.IsNullOrWhiteSpace(longName))
continue;
Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}");
await DownloadSinglePack(url, system, outDir, subfolder);
}
Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}");
Console.WriteLine();
return true;
}
/// <summary>
/// Download from a URI to a local file
/// </summary>
/// <param name="uri">Remote URI to retrieve</param>
/// <param name="fileName">Filename to write to</param>
/// <returns>The remote filename from the URI, null on error</returns>
private async Task<string> DownloadFile(string uri, string fileName)
{
// Make the call to get the file
var response = await GetAsync(uri);
if (response?.Content?.Headers == null || !response.IsSuccessStatusCode)
{
Console.WriteLine($"Could not download {uri}");
return null;
}
// Copy the data to a local temp file
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (var tempFileStream = File.OpenWrite(fileName))
{
responseStream.CopyTo(tempFileStream);
}
return response.Content.Headers.ContentDisposition?.FileName?.Replace("\"", "");
}
/// <summary>
/// Download from a URI to a string
/// </summary>
/// <param name="uri">Remote URI to retrieve</param>
/// <param name="retries">Number of times to retry on error</param>
/// <returns>String from the URI, null on error</returns>
private async Task<string> DownloadString(string uri, int retries = 3)
{
// Only retry a positive number of times
if (retries <= 0)
return null;
for (int i = 0; i < retries; i++)
{
try
{
return await GetStringAsync(uri);
}
catch { }
}
return null;
}
/// <summary>
/// Move a tempfile to a new name unless it aleady exists, in which case, delete the tempfile
/// </summary>
/// <param name="tempfile">Path to existing temporary file</param>
/// <param name="newfile">Path to new output file</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="subfolder">Optional subfolder to append to the path</param>
private static void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder)
{
// If we don't have a file to move to, just delete the temp file
if (string.IsNullOrWhiteSpace(newfile))
{
File.Delete(tempfile);
return;
}
// If we have a subfolder, create it and update the newfile name
if (!string.IsNullOrWhiteSpace(subfolder))
{
if (!Directory.Exists(Path.Combine(outDir, subfolder)))
Directory.CreateDirectory(Path.Combine(outDir, subfolder));
newfile = Path.Combine(subfolder, newfile);
}
// If the file already exists, don't overwrite it
if (File.Exists(Path.Combine(outDir, newfile)))
File.Delete(tempfile);
else
File.Move(tempfile, Path.Combine(outDir, newfile));
}
#endregion
}
}
#endif

31
RedumpLib/Web/RedumpWebClient.cs
View File

@@ -1,4 +1,6 @@
using System;
#if NET48 || NETSTANDARD2_1
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
@@ -32,25 +34,26 @@ namespace RedumpLib.Web
// If the response headers are null or empty
if (ResponseHeaders == null || ResponseHeaders.Count == 0)
return null;
// If we don't have the response header we care about
string headerValue = ResponseHeaders.Get("Content-Disposition");
if (string.IsNullOrWhiteSpace(headerValue))
return null;
// Extract the filename from the value
#if NETSTANDARD2_1
return headerValue[(headerValue.IndexOf("filename=") + 9)..].Replace("\"", "");
#else
return headerValue.Substring(headerValue.IndexOf("filename=") + 9).Replace("\"", "");
#endif
}
/// <inheritdoc/>
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest request = base.GetWebRequest(address);
HttpWebRequest webRequest = request as HttpWebRequest;
if (webRequest != null)
{
if (request is HttpWebRequest webRequest)
webRequest.CookieContainer = m_container;
}
return request;
}
@@ -65,8 +68,12 @@ namespace RedumpLib.Web
return (false, null);
// Try logging in with the supplied credentials otherwise
#if NETSTANDARD2_1
using RedumpWebClient wc = new RedumpWebClient();
#else
using (RedumpWebClient wc = new RedumpWebClient())
{
#endif
bool? loggedIn = wc.Login(username, password);
if (loggedIn == true)
return (true, "Redump username and password accepted!");
@@ -74,7 +81,9 @@ namespace RedumpLib.Web
return (false, "Redump username and password denied!");
else
return (null, "An error occurred validating your credentials!");
#if NET48
}
#endif
}
/// <summary>
@@ -524,7 +533,7 @@ namespace RedumpLib.Web
// View Edit History
if (discPage.Contains($"<a href=\"/disc/{id}/changes/\""))
DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.ChangesExt, Path.Combine(paddedIdDir, "changes.html"));
// CUE
if (discPage.Contains($"<a href=\"/disc/{id}/cue/\""))
DownloadFile(string.Format(Constants.DiscPageUrl, +id) + Constants.CueExt, Path.Combine(paddedIdDir, paddedId + ".cue"));
@@ -741,7 +750,7 @@ namespace RedumpLib.Web
// If we didn't have credentials
if (!LoggedIn && system.IsBanned())
continue;
// If the system is unknown, we can't do anything
string longName = system.LongName();
if (string.IsNullOrWhiteSpace(longName))
@@ -787,7 +796,7 @@ namespace RedumpLib.Web
Console.Write($"\r{longName}{new string(' ', Console.BufferWidth - longName.Length - 1)}");
DownloadSinglePack(url, system, outDir, subfolder);
}
}
Console.Write($"\rComplete!{new string(' ', Console.BufferWidth - 10)}");
Console.WriteLine();
@@ -800,7 +809,7 @@ namespace RedumpLib.Web
/// <param name="newfile">Path to new output file</param>
/// <param name="outDir">Output directory to save data to</param>
/// <param name="subfolder">Optional subfolder to append to the path</param>
private void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder)
private static void MoveOrDelete(string tempfile, string newfile, string outDir, string subfolder)
{
if (!string.IsNullOrWhiteSpace(newfile))
{
@@ -824,3 +833,5 @@ namespace RedumpLib.Web
#endregion
}
}
#endif