Add ability to get matching IDs from Redump

This commit is contained in:
Matt Nadareski
2019-05-14 01:46:10 -07:00
parent e61ca31a3f
commit 2787377250
11 changed files with 617 additions and 4 deletions

View File

@@ -57,12 +57,22 @@ namespace DICUI.Check
return;
}
// Check for Redump login credentials
string username = null, password = null;
int startIndex = 2;
if (args[2] == "-c" || args[2] == "--credentials")
{
username = args[3];
password = args[4];
startIndex = 5;
}
// Make a new Progress object
var progress = new Progress<Result>();
progress.ProgressChanged += ProgressUpdated;
// Loop through all the rest of the args
for (int i = 2; i < args.Length; i++)
for (int i = startIndex; i < args.Length; i++)
{
// Check for a file
if (!File.Exists(args[i]))
@@ -82,6 +92,9 @@ namespace DICUI.Check
System = knownSystem,
Type = mediaType,
ScanForProtection = false,
Username = username,
Password = password,
};
env.FixOutputPaths();
@@ -101,7 +114,7 @@ namespace DICUI.Check
Console.WriteLine(error);
Console.WriteLine("Usage:");
Console.WriteLine("DICUI.Check.exe <mediatype> <system> </path/to/output.bin> ...");
Console.WriteLine("DICUI.Check.exe <mediatype> <system> [-c username password] </path/to/output.bin> ...");
Console.WriteLine();
Console.WriteLine(@"Common Media Types:\r\n
bd / bluray - BD-ROM

View File

@@ -63,6 +63,9 @@
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Web\CookieAwareWebClient.cs">
<SubType>Component</SubType>
</Compile>
<Compile Include="Data\Constants.cs" />
<Compile Include="Data\Enumerations.cs" />
<Compile Include="Data\SubmissionInfo.cs" />
@@ -73,6 +76,7 @@
<Compile Include="Utilities\Parameters.cs" />
<Compile Include="Utilities\Result.cs" />
<Compile Include="Utilities\Validators.cs" />
<Compile Include="Web\RedumpAccess.cs" />
</ItemGroup>
<ItemGroup>
<Content Include="mspack.dll">

View File

@@ -100,6 +100,7 @@
public const string CRC32Field = "CRC32";
public const string MD5Field = "MD5";
public const string SHA1Field = "SHA1";
public const string MatchingIDsField = "Matching IDs";
public const string ErrorCountField = "Error Count";
public const string CuesheetField = "Cuesheet";
public const string SubIntentionField = "SubIntention Data (SecuROM/LibCrypt)";

View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
@@ -240,5 +241,12 @@ namespace DICUI.Data
public string SHA1 { get; set; }
#endregion
#region Nonstandard Information
[JsonIgnore]
public List<int> MatchedIDs { get; set; }
#endregion
}
}

View File

@@ -1465,6 +1465,44 @@ namespace DICUI.Utilities
#region Convert From String
/// <summary>
/// Get the Category enum value for a given string
/// </summary>
/// <param name="sys">String value to convert</param>
/// <returns>Category represented by the string, if possible</returns>
public static Category StringToCategory(string category)
{
switch (category.ToLowerInvariant())
{
case "games":
return Category.Games;
case "demos":
return Category.Demos;
case "video":
return Category.Video;
case "audio":
return Category.Audio;
case "multimedia":
return Category.Multimedia;
case "applications":
return Category.Applications;
case "coverdiscs":
return Category.Coverdiscs;
case "educational":
return Category.Educational;
case "bonusdiscs":
case "bonus discs":
return Category.BonusDiscs;
case "preproduction":
return Category.Preproduction;
case "addons":
case "add-ons":
return Category.AddOns;
default:
return Category.Games;
}
}
/// <summary>
/// Get the KnownSystem enum value for a given string
/// </summary>
@@ -2175,6 +2213,92 @@ namespace DICUI.Utilities
}
}
/// <summary>
/// Get the Language enum value for a given string
/// </summary>
/// <param name="sys">String value to convert</param>
/// <returns>Language represented by the string, if possible</returns>
public static Language? StringToLanguage(string lang)
{
switch (lang)
{
case "afr":
return Language.Afrikaans;
case "ara":
return Language.Arabic;
case "baq":
return Language.Basque;
case "bul":
return Language.Bulgarian;
case "cat":
return Language.Catalan;
case "chi":
return Language.Chinese;
case "hrv":
return Language.Croatian;
case "cze":
return Language.Czech;
case "dan":
return Language.Danish;
case "dut":
return Language.Dutch;
case "eng":
return Language.English;
case "fin":
return Language.Finnish;
case "fre":
return Language.French;
case "gla":
return Language.Gaelic;
case "ger":
return Language.German;
case "gre":
return Language.Greek;
case "heb":
return Language.Hebrew;
case "hin":
return Language.Hindi;
case "hun":
return Language.Hungarian;
case "ita":
return Language.Italian;
case "jap":
return Language.Japanese;
case "kor":
return Language.Korean;
case "nor":
return Language.Norwegian;
case "pol":
return Language.Polish;
case "por":
return Language.Portuguese;
case "pan":
return Language.Punjabi;
case "ron":
return Language.Romanian;
case "rus":
return Language.Russian;
case "slk":
return Language.Slovak;
case "slv":
return Language.Slovenian;
case "spa":
return Language.Spanish;
case "swe":
return Language.Swedish;
case "tam":
return Language.Tamil;
case "tha":
return Language.Thai;
case "tur":
return Language.Turkish;
case "ukr":
return Language.Ukrainian;
default:
return null;
}
}
/// <summary>
/// Get the MediaType enum value for a given string
/// </summary>
@@ -2348,6 +2472,138 @@ namespace DICUI.Utilities
}
}
/// <summary>
/// Get the Region enum value for a given string
/// </summary>
/// <param name="type">String value to convert</param>
/// <returns>Region represented by the string, if possible</returns>
public static Region? StringToRegion(string region)
{
switch (region)
{
case "Ar":
return Region.Argentina;
case "A":
return Region.Asia;
case "A,E":
return Region.AsiaEurope;
case "A,U":
return Region.AsiaUSA;
case "Au":
return Region.Australia;
case "At":
return Region.Austria;
case "At,Ch":
return Region.AustriaSwitzerland;
case "Be":
return Region.Belgium;
case "Be,N":
return Region.BelgiumNetherlands;
case "B":
return Region.Brazil;
case "Ca":
return Region.Canada;
case "C":
return Region.China;
case "Hr":
return Region.Croatia;
case "Cz":
return Region.Czech;
case "Dk":
return Region.Denmark;
case "E":
return Region.Europe;
case "E,A":
return Region.EuropeAsia;
case "E,Au":
return Region.EuropeAustralia;
case "Fi":
return Region.Finland;
case "F":
return Region.France;
case "F,S":
return Region.FranceSpain;
case "G":
return Region.Germany;
case "Gr":
return Region.Greece;
case "H":
return Region.Hungary;
case "In":
return Region.India;
case "Ie":
return Region.Ireland;
case "Il":
return Region.Israel;
case "I":
return Region.Italy;
case "J":
return Region.Japan;
case "J,A":
return Region.JapanAsia;
case "J,E":
return Region.JapanEurope;
case "J,K":
return Region.JapanKorea;
case "J,U":
return Region.JapanUSA;
case "K":
return Region.Korea;
case "LAm":
return Region.LatinAmerica;
case "N":
return Region.Netherlands;
case "No":
return Region.Norway;
case "P":
return Region.Poland;
case "Pt":
return Region.Portugal;
case "R":
return Region.Russia;
case "Sca":
return Region.Scandinavia;
case "Sg":
return Region.Singapore;
case "Sk":
return Region.Slovakia;
case "Za":
return Region.SouthAfrica;
case "S":
return Region.Spain;
case "Sw":
return Region.Sweden;
case "Ch":
return Region.Switzerland;
case "Tw":
return Region.Taiwan;
case "Th":
return Region.Thailand;
case "Tr":
return Region.Turkey;
case "Ae":
return Region.UnitedArabEmirates;
case "Uk":
return Region.UK;
case "Ue":
return Region.Ukraine;
case "U":
return Region.USA;
case "U,A":
return Region.USAAsia;
case "U,B":
return Region.USABrazil;
case "U,E":
return Region.USAEurope;
case "U,J":
return Region.USAJapan;
case "W":
return Region.World;
default:
return null;
}
}
#endregion
}
}

View File

@@ -7,6 +7,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DICUI.Data;
using DICUI.Web;
using Newtonsoft.Json;
namespace DICUI.Utilities
@@ -59,6 +60,10 @@ namespace DICUI.Utilities
public bool ScanForProtection;
public int RereadAmountC2;
// Redump login information
public string Username;
public string Password;
// External process information
private Process dicProcess;
@@ -447,6 +452,112 @@ namespace DICUI.Utilities
ClrMameProData = GetDatfile(combinedBase + ".dat"),
};
// First and foremost, we want to get a list of matching IDs for each line in the DAT
if (!string.IsNullOrEmpty(info.ClrMameProData) && !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password))
{
info.MatchedIDs = new List<int>();
using (CookieAwareWebClient wc = new CookieAwareWebClient())
{
// Login to Redump
RedumpAccess access = new RedumpAccess();
if (access.RedumpLogin(wc, this.Username, this.Password))
{// Loop through all of the hashdata to find matching IDs
progress?.Report(Result.Success("Finding disc matches on Redump..."));
string[] splitData = info.ClrMameProData.Split('\n');
foreach (string hashData in splitData)
{
if (GetISOHashValues(hashData, out long size, out string crc32, out string md5, out string sha1))
{
List<int> newIds = access.ProcessSearch(wc, sha1);
if (info.MatchedIDs.Any())
info.MatchedIDs = info.MatchedIDs.Intersect(newIds).ToList();
else
info.MatchedIDs = newIds;
}
}
progress?.Report(Result.Success("Match finding complete! " + (info.MatchedIDs.Count > 0 ? "Matched IDs: " + string.Join(",", info.MatchedIDs) : "No matches found")));
// If we have exactly 1 ID, we can grab a bunch of info from it
if (info.MatchedIDs.Count == 1)
{
progress?.Report(Result.Success($"Filling fields from existing ID {info.MatchedIDs[0]}..."));
string discData = access.DownloadSingleSiteID(wc, info.MatchedIDs[0]);
// Title, Disc Number/Letter, Disc Title
var match = Regex.Match(discData, @"<h1>(.*?)</h1>");
if (match.Success)
info.Title = match.Groups[1].Value;
// Foreign Title
match = Regex.Match(discData, @"<h2>(.*?)</h2>");
if (match.Success)
info.ForeignTitleNonLatin = match.Groups[1].Value;
else
info.ForeignTitleNonLatin = null;
// Category
match = Regex.Match(discData, @"<tr><th>Category</th><td>(.*?)</td></tr>");
if (match.Success)
info.Category = Converters.StringToCategory(match.Groups[1].Value);
// Region
match = Regex.Match(discData, @"<tr><th>Region</th><td><a href=""/discs/region/(.*?)/"">");
if (match.Success)
info.Region = Converters.StringToRegion(match.Groups[1].Value);
// Languages
var matches = Regex.Matches(discData, @"<img src=""/images/languages/(.*?)\.png"" alt="".*?"" title="".*?"" />\s*");
if (matches.Count > 0)
{
List<Language?> tempLanguages = new List<Language?>();
foreach (Match submatch in matches)
tempLanguages.Add(Converters.StringToLanguage(submatch.Groups[1].Value));
info.Languages = tempLanguages.ToArray();
}
// Serial
match = Regex.Match(discData, @"<tr><th>Serial</th><td>(.*?)</td></tr>");
if (match.Success)
info.Serial = match.Groups[1].Value;
// Version
match = Regex.Match(discData, @"<tr><th>Version</th><td>(.*?)</td></tr>");
if (match.Success)
info.Version = match.Groups[1].Value;
// Edition
match = Regex.Match(discData, @"<tr><th>Edition</th><td>(.*?)</td></tr>");
if (match.Success)
info.OtherEditions = match.Groups[1].Value;
// Comments
match = Regex.Match(discData, @"<tr><th>Comments</th></tr><tr><td>(.*?)</td></tr>");
if (match.Success)
{
info.Comments = match.Groups[1].Value
.Replace("<br />", "\n")
.Replace("<b>ISBN</b>", "[T:ISBN]") + "\n";
}
// Contents
match = Regex.Match(discData, @"<tr><th>Contents</th></tr><tr .*?><td>(.*?)</td></tr>");
if (match.Success)
{
info.Contents = match.Groups[1].Value
.Replace("<br />", "\n")
.Replace("</div>", "");
info.Contents = Regex.Replace(info.Contents, @"<div .*?>", "");
}
progress?.Report(Result.Success("Information filling complete!"));
}
}
}
}
// Now we want to do a check by MediaType and extract all required info
switch (Type)
{
@@ -484,7 +595,8 @@ namespace DICUI.Utilities
case KnownSystem.EnhancedCD:
case KnownSystem.IBMPCCompatible:
case KnownSystem.RainbowDisc:
info.Comments += $"[T:ISBN] {Template.OptionalValue}";
if (string.IsNullOrWhiteSpace(info.Comments))
info.Comments += $"[T:ISBN] {Template.OptionalValue}";
progress?.Report(Result.Success("Running copy protection scan... this might take a while!"));
info.Protection = GetCopyProtection();
@@ -630,7 +742,8 @@ namespace DICUI.Utilities
case KnownSystem.EnhancedCD:
case KnownSystem.IBMPCCompatible:
case KnownSystem.RainbowDisc:
info.Comments += $"[T:ISBN] {Template.OptionalValue}";
if (string.IsNullOrWhiteSpace(info.Comments))
info.Comments += $"[T:ISBN] {Template.OptionalValue}";
progress?.Report(Result.Success("Running copy protection scan... this might take a while!"));
info.Protection = GetCopyProtection();
@@ -760,6 +873,7 @@ namespace DICUI.Utilities
AddIfExists(output, Template.SystemField, info.System.Name(), 1);
AddIfExists(output, Template.MediaTypeField, GetFixedMediaType(info.Media, info.Layerbreak), 1);
AddIfExists(output, Template.CategoryField, info.Category.Name(), 1);
AddIfExists(output, Template.MatchingIDsField, info.MatchedIDs, 1);
AddIfExists(output, Template.RegionField, info.Region.Name(), 1);
AddIfExists(output, Template.LanguagesField, (info.Languages ?? new Language?[] { null }).Select(l => l.Name()).ToArray(), 1);
AddIfExists(output, Template.PlaystationLanguageSelectionViaField, info.LanguageSelection, 1);
@@ -947,6 +1061,22 @@ namespace DICUI.Utilities
AddIfExists(output, key, string.Join(", ", value), indent);
}
/// <summary>
/// Add the properly formatted key and value, if possible
/// </summary>
/// <param name="output">Output list</param>
/// <param name="key">Name of the output key to write</param>
/// <param name="value">Name of the output value to write</param>
/// <param name="indent">Number of tabs to indent the line</param>
private void AddIfExists(List<string> output, string key, List<int> value, int indent)
{
// If there's no valid value to write
if (value == null || value.Count() == 0)
return;
AddIfExists(output, key, string.Join(", ", value.Select(o => o.ToString())), indent);
}
/// <summary>
/// Get the existance of an anti-modchip string from the input file, if possible
/// </summary>

View File

@@ -0,0 +1,36 @@
using System;
using System.Net;
namespace DICUI.Web
{
// https://stackoverflow.com/questions/1777221/using-cookiecontainer-with-webclient-class
public class CookieAwareWebClient : WebClient
{
private readonly CookieContainer m_container = new CookieContainer();
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest request = base.GetWebRequest(address);
HttpWebRequest webRequest = request as HttpWebRequest;
if (webRequest != null)
{
webRequest.CookieContainer = m_container;
}
return request;
}
/// <summary>
/// Get the last downloaded filename, if possible
/// </summary>
/// <returns></returns>
public string GetLastFilename()
{
// Try to extract the filename from the Content-Disposition header
if (!String.IsNullOrEmpty(this.ResponseHeaders["Content-Disposition"]))
return this.ResponseHeaders["Content-Disposition"].Substring(this.ResponseHeaders["Content-Disposition"].IndexOf("filename=") + 9).Replace("\"", "");
return null;
}
}
}

View File

@@ -0,0 +1,154 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
namespace DICUI.Web
{
public class RedumpAccess
{
#region Base URLs
private const string loginUrl = "http://forum.redump.org/login/";
#endregion
#region Disc URLs and Extensions
private const string discPageUrl = @"http://redump.org/disc/{0}/";
#endregion
#region List URLs
private const string quickSearchUrl = @"http://redump.org/discs/quicksearch/{0}/?page={1}";
#endregion
#region Regexes
private Regex discRegex = new Regex(@"<a href=""/disc/(\d+)/"">");
private Regex tokenRegex = new Regex(@"<input type=""hidden"" name=""csrf_token"" value=""(.*?)"" />");
#endregion
/// <summary>
/// Login to Redump, if possible
/// </summary>
/// <param name="wc">CookieAwareWebClient to hold the login state</param>
/// <param name="username">Redump username to log in for protected systems</param>
/// <param name="password">Redump password to log in for protected systems</param>
/// <returns>True login was successful, false otherwise</returns>
public bool RedumpLogin(CookieAwareWebClient wc, string username, string password)
{
if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
return false;
var loginPage = wc.DownloadString(loginUrl);
string token = this.tokenRegex.Match(loginPage).Groups[1].Value;
wc.Headers[HttpRequestHeader.ContentType] = "application/x-www-form-urlencoded";
wc.Encoding = Encoding.UTF8;
var response = wc.UploadString(loginUrl, $"form_sent=1&redirect_url=&csrf_token={token}&req_username={username}&req_password={password}&save_pass=0");
if (response.Contains("Incorrect username and/or password."))
return false;
else
return true;
}
/// <summary>
/// Get the list of IDs that associate with a given string
/// </summary>
/// <param name="wc">CookieAwareWebClient to hold the login state</param>
/// <param name="query">Value to search for in Redump</param>
/// <returns>A list of IDs associated with that value</returns>
public List<int> ProcessSearch(CookieAwareWebClient wc, string query)
{
List<int> ids = new List<int>();
// Keep getting quicksearch pages until there are none left
int pageNumber = 1;
while (true)
{
List<int> pageIds = CheckSingleSitePage(wc, string.Format(quickSearchUrl, query, pageNumber++));
ids.AddRange(pageIds);
if (pageIds.Count < 2)
break;
}
return ids;
}
/// <summary>
/// Process a Redump site page as a list of possible IDs or disc page
/// </summary>
/// <param name="wc">CookieAwareWebClient to access the pages</param>
/// <param name="url">Page URL to check and parse</param>
/// <returns>List of matching IDs</returns>
private List<int> CheckSingleSitePage(CookieAwareWebClient wc, string url)
{
List<int> ids = new List<int>();
var dumpsPage = wc.DownloadString(url);
// If we have no dumps left
if (dumpsPage.Contains("No discs found."))
return ids;
// If we have a single disc page already
if (dumpsPage.Contains("<b>Download:</b>"))
{
var value = Regex.Match(dumpsPage, @"/disc/(\d+)/sfv/").Groups[1].Value;
if (Int32.TryParse(value, out int id))
ids.Add(id);
return ids;
}
// Otherwise, traverse each dump on the page
var matches = discRegex.Matches(dumpsPage);
foreach (Match match in matches)
{
try
{
if (Int32.TryParse(match.Groups[1].Value, out int value))
ids.Add(value);
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
continue;
}
}
return ids;
}
/// <summary>
/// Download an individual site ID page as a string, if possible
/// </summary>
/// <param name="wc">CookieAwareWebClient to access the pages</param>
/// <param name="id">Redump disc ID to retrieve</param>
/// <returns></returns>
public string DownloadSingleSiteID(CookieAwareWebClient wc, int id)
{
string paddedId = id.ToString().PadLeft(5, '0');
Console.WriteLine($"Processing ID: {paddedId}");
try
{
string discPage = wc.DownloadString(string.Format(discPageUrl, +id));
if (discPage.Contains($"Disc with ID \"{id}\" doesn't exist"))
return null;
else
return discPage;
}
catch (Exception ex)
{
Console.WriteLine($"An exception has occurred: {ex}");
return null;
}
}
}
}

View File

@@ -10,5 +10,7 @@
<add key="RereadAmountForC2" value="20"/>
<add key="VerboseLogging" value="true"/>
<add key="OpenLogWindowAtStartup" value="true"/>
<add key="username" value=""/>
<add key="password" value=""/>
</appSettings>
</configuration>

View File

@@ -25,6 +25,9 @@ namespace DICUI
public bool VerboseLogging { get; set; }
public bool OpenLogWindowAtStartup { get; set; }
public string Username { get; set; }
public string Password { get; set; }
public void Save()
{
Configuration configFile = ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None);
@@ -60,6 +63,9 @@ namespace DICUI
this.RereadAmountForC2 = Int32.TryParse(ConfigurationManager.AppSettings["RereadAmountForC2"], out int rereadAmountForC2) ? rereadAmountForC2 : 20;
this.VerboseLogging = Boolean.TryParse(ConfigurationManager.AppSettings["VerboseLogging"], out bool verboseLogging) ? verboseLogging : true;
this.OpenLogWindowAtStartup = Boolean.TryParse(ConfigurationManager.AppSettings["OpenLogWindowAtStartup"], out bool openLogWindowAtStartup) ? openLogWindowAtStartup : true;
this.Username = ConfigurationManager.AppSettings["username"] ?? "";
this.Password = ConfigurationManager.AppSettings["password"] ?? "";
}

View File

@@ -387,6 +387,9 @@ namespace DICUI.Windows
ScanForProtection = _options.ScanForProtection,
RereadAmountC2 = _options.RereadAmountForC2,
Username = _options.Username,
Password = _options.Password,
System = SystemTypeComboBox.SelectedItem as KnownSystemComboBoxItem,
Type = MediaTypeComboBox.SelectedItem as MediaType?,
};