mirror of
https://github.com/claunia/SabreTools.git
synced 2025-12-16 19:14:27 +00:00
Move most filtering up a layer, add TODO for tests
This commit is contained in:
Matt Nadareski
@@ -5,7 +5,6 @@ using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
using System.Threading.Tasks;
|
||||
#endif
|
||||
@@ -1242,247 +1241,6 @@ namespace SabreTools.DatFiles
|
||||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use game descriptions as names, updating cloneof/romof/sampleof
|
||||
/// </summary>
|
||||
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
|
||||
internal void MachineDescriptionToName(bool throwOnError = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
// First we want to get a mapping for all games to description
|
||||
var mapping = CreateMachineToDescriptionMapping();
|
||||
|
||||
// Now we loop through every item and update accordingly
|
||||
UpdateMachineNamesFromDescriptions(mapping);
|
||||
}
|
||||
catch (Exception ex) when (!throwOnError)
|
||||
{
|
||||
_logger.Warning(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensure that all roms are in their own game (or at least try to ensure)
|
||||
/// </summary>
|
||||
internal void SetOneRomPerGame()
|
||||
{
|
||||
// For each rom, we want to update the game to be "<game name>/<rom name>"
|
||||
#if NET452_OR_GREATER || NETCOREAPP
|
||||
Parallel.ForEach(SortedKeys, Core.Globals.ParallelOptions, key =>
|
||||
#elif NET40_OR_GREATER
|
||||
Parallel.ForEach(SortedKeys, key =>
|
||||
#else
|
||||
foreach (var key in SortedKeys)
|
||||
#endif
|
||||
{
|
||||
var items = GetItemsForBucket(key);
|
||||
if (items == null)
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
return;
|
||||
#else
|
||||
continue;
|
||||
#endif
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
SetOneRomPerGame(item);
|
||||
}
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
});
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Filter a DAT using 1G1R logic given an ordered set of regions
|
||||
/// </summary>
|
||||
/// <param name="regionList">List of regions in order of priority</param>
|
||||
/// <remarks>
|
||||
/// In the most technical sense, the way that the region list is being used does not
|
||||
/// confine its values to be just regions. Since it's essentially acting like a
|
||||
/// specialized version of the machine name filter, anything that is usually encapsulated
|
||||
/// in parenthesis would be matched on, including disc numbers, languages, editions,
|
||||
/// and anything else commonly used. Please note that, unlike other existing 1G1R
|
||||
/// solutions, this does not have the ability to contain custom mappings of parent
|
||||
/// to clone sets based on name, nor does it have the ability to match on the
|
||||
/// Release DatItem type.
|
||||
/// </remarks>
|
||||
internal void SetOneGamePerRegion(List<string> regionList)
|
||||
{
|
||||
// If we have null region list, make it empty
|
||||
regionList ??= [];
|
||||
|
||||
// For sake of ease, the first thing we want to do is bucket by game
|
||||
BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
|
||||
|
||||
// Then we want to get a mapping of all machines to parents
|
||||
Dictionary<string, List<string>> parents = [];
|
||||
foreach (string key in SortedKeys)
|
||||
{
|
||||
var items = GetItemsForBucket(key);
|
||||
if (items == null || items.Count == 0)
|
||||
continue;
|
||||
|
||||
var item = items.First();
|
||||
var machine = GetMachineForItem(item.Key);
|
||||
if (machine.Value == null)
|
||||
continue;
|
||||
|
||||
// Get machine information
|
||||
Machine? machineObj = machine.Value.GetFieldValue<Machine>(DatItem.MachineKey);
|
||||
string? machineName = machineObj?.GetStringFieldValue(Models.Metadata.Machine.NameKey)?.ToLowerInvariant();
|
||||
if (machineObj == null || machineName == null)
|
||||
continue;
|
||||
|
||||
// Get the string values
|
||||
string? cloneOf = machineObj.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)?.ToLowerInvariant();
|
||||
string? romOf = machineObj.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)?.ToLowerInvariant();
|
||||
|
||||
// Match on CloneOf first
|
||||
if (!string.IsNullOrEmpty(cloneOf))
|
||||
{
|
||||
if (!parents.ContainsKey(cloneOf!))
|
||||
parents.Add(cloneOf!, []);
|
||||
|
||||
parents[cloneOf!].Add(machineName);
|
||||
}
|
||||
|
||||
// Then by RomOf
|
||||
else if (!string.IsNullOrEmpty(romOf))
|
||||
{
|
||||
if (!parents.ContainsKey(romOf!))
|
||||
parents.Add(romOf!, []);
|
||||
|
||||
parents[romOf!].Add(machineName);
|
||||
}
|
||||
|
||||
// Otherwise, treat it as a parent
|
||||
else
|
||||
{
|
||||
if (!parents.ContainsKey(machineName))
|
||||
parents.Add(machineName, []);
|
||||
|
||||
parents[machineName].Add(machineName);
|
||||
}
|
||||
}
|
||||
|
||||
// Once we have the full list of mappings, filter out games to keep
|
||||
foreach (string key in parents.Keys)
|
||||
{
|
||||
// Find the first machine that matches the regions in order, if possible
|
||||
string? machine = default;
|
||||
foreach (string region in regionList)
|
||||
{
|
||||
machine = parents[key].Find(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase));
|
||||
if (machine != default)
|
||||
break;
|
||||
}
|
||||
|
||||
// If we didn't get a match, use the parent
|
||||
if (machine == default)
|
||||
machine = key;
|
||||
|
||||
// Remove the key from the list
|
||||
parents[key].Remove(machine);
|
||||
|
||||
// Remove the rest of the items from this key
|
||||
parents[key].ForEach(k => RemoveMachine(k));
|
||||
}
|
||||
|
||||
// Finally, strip out the parent tags
|
||||
RemoveTagsFromChild();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Strip the dates from the beginning of scene-style set names
|
||||
/// </summary>
|
||||
internal void StripSceneDatesFromItems()
|
||||
{
|
||||
// Set the regex pattern to use
|
||||
const string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)";
|
||||
|
||||
// Now process all of the machines
|
||||
#if NET452_OR_GREATER || NETCOREAPP
|
||||
Parallel.ForEach(GetMachines(), Core.Globals.ParallelOptions, machine =>
|
||||
#elif NET40_OR_GREATER
|
||||
Parallel.ForEach(GetMachines(), machine =>
|
||||
#else
|
||||
foreach (var machine in GetMachines())
|
||||
#endif
|
||||
{
|
||||
// Get the current machine
|
||||
if (machine.Value == null)
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
return;
|
||||
#else
|
||||
continue;
|
||||
#endif
|
||||
|
||||
if (Regex.IsMatch(machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)!, pattern))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.NameKey, Regex.Replace(machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)!, pattern, "$2"));
|
||||
|
||||
if (Regex.IsMatch(machine.Value.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!, pattern))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.DescriptionKey, Regex.Replace(machine.Value.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!, pattern, "$2"));
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
});
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create machine to description mapping dictionary
|
||||
/// </summary>
|
||||
private IDictionary<string, string> CreateMachineToDescriptionMapping()
|
||||
{
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
ConcurrentDictionary<string, string> mapping = new();
|
||||
#else
|
||||
Dictionary<string, string> mapping = [];
|
||||
#endif
|
||||
#if NET452_OR_GREATER || NETCOREAPP
|
||||
Parallel.ForEach(GetMachines(), Core.Globals.ParallelOptions, machine =>
|
||||
#elif NET40_OR_GREATER
|
||||
Parallel.ForEach(GetMachines(), machine =>
|
||||
#else
|
||||
foreach (var machine in GetMachines())
|
||||
#endif
|
||||
{
|
||||
// Get the current machine
|
||||
if (machine.Value == null)
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
return;
|
||||
#else
|
||||
continue;
|
||||
#endif
|
||||
|
||||
// Get the values to check against
|
||||
string? machineName = machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey);
|
||||
string? machineDesc = machine.Value.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey);
|
||||
if (machineName == null || machineDesc == null)
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
return;
|
||||
#else
|
||||
continue;
|
||||
#endif
|
||||
|
||||
// If the key mapping doesn't exist, add it
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
mapping.TryAdd(machineName, machineDesc.Replace('/', '_').Replace("\"", "''").Replace(":", " -"));
|
||||
#else
|
||||
mapping[machineName] = machineDesc.Replace('/', '_').Replace("\"", "''").Replace(":", " -");
|
||||
#endif
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
});
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
|
||||
return mapping;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Execute all filters in a filter runner on a single bucket
|
||||
/// </summary>
|
||||
@@ -1506,127 +1264,6 @@ namespace SabreTools.DatFiles
|
||||
_buckets[bucketName] = newItems;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set internal names to match One Rom Per Game (ORPG) logic
|
||||
/// </summary>
|
||||
/// <param name="datItem">DatItem to run logic on</param>
|
||||
private void SetOneRomPerGame(KeyValuePair<long, DatItem> datItem)
|
||||
{
|
||||
// If the item name is null
|
||||
string? machineName = datItem.Value.GetName();
|
||||
if (datItem.Key < 0 || machineName == null)
|
||||
return;
|
||||
|
||||
// Get the current machine
|
||||
var machine = GetMachineForItem(datItem.Key);
|
||||
if (machine.Value == null)
|
||||
return;
|
||||
|
||||
// Remove extensions from Rom items
|
||||
if (datItem.Value is Rom)
|
||||
{
|
||||
string[] splitname = machineName.Split('.');
|
||||
machineName = machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)
|
||||
+ $"/{string.Join(".", splitname, 0, splitname.Length > 1 ? splitname.Length - 1 : 1)}";
|
||||
}
|
||||
|
||||
// Strip off "Default" prefix only for ORPG
|
||||
if (machineName.StartsWith("Default"))
|
||||
machineName = machineName.Substring("Default".Length + 1);
|
||||
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.NameKey, machineName);
|
||||
datItem.Value.SetName(Path.GetFileName(datItem.Value.GetName()));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Update machine names from descriptions according to mappings
|
||||
/// </summary>
|
||||
private void UpdateMachineNamesFromDescriptions(IDictionary<string, string> mapping)
|
||||
{
|
||||
#if NET452_OR_GREATER || NETCOREAPP
|
||||
Parallel.ForEach(SortedKeys, Core.Globals.ParallelOptions, key =>
|
||||
#elif NET40_OR_GREATER
|
||||
Parallel.ForEach(SortedKeys, key =>
|
||||
#else
|
||||
foreach (var key in SortedKeys)
|
||||
#endif
|
||||
{
|
||||
var items = GetItemsForBucket(key);
|
||||
if (items == null)
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
return;
|
||||
#else
|
||||
continue;
|
||||
#endif
|
||||
|
||||
List<long> newItems = [];
|
||||
foreach (var item in items)
|
||||
{
|
||||
// Get the current machine
|
||||
var machine = GetMachineForItem(item.Key);
|
||||
if (machine.Value == null)
|
||||
continue;
|
||||
|
||||
// Update machine name
|
||||
if (!string.IsNullOrEmpty(machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)) && mapping.ContainsKey(machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)!))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.NameKey, mapping[machine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey)!]);
|
||||
|
||||
// Update cloneof
|
||||
if (!string.IsNullOrEmpty(machine.Value.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)) && mapping.ContainsKey(machine.Value.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.CloneOfKey, mapping[machine.Value.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!]);
|
||||
|
||||
// Update romof
|
||||
if (!string.IsNullOrEmpty(machine.Value.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)) && mapping.ContainsKey(machine.Value.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.RomOfKey, mapping[machine.Value.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!]);
|
||||
|
||||
// Update sampleof
|
||||
if (!string.IsNullOrEmpty(machine.Value.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)) && mapping.ContainsKey(machine.Value.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!))
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.SampleOfKey, mapping[machine.Value.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!]);
|
||||
|
||||
// Add the new item to the output list
|
||||
newItems.Add(item.Key);
|
||||
}
|
||||
|
||||
// Replace the old list of roms with the new one
|
||||
_buckets[key] = newItems;
|
||||
#if NET40_OR_GREATER || NETCOREAPP
|
||||
});
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
// TODO: All internal, can this be put into a better location?
|
||||
#region Splitting
|
||||
|
||||
/// <summary>
|
||||
/// Remove all romof and cloneof tags from all games
|
||||
/// </summary>
|
||||
internal void RemoveTagsFromChild()
|
||||
{
|
||||
List<string> games = [.. SortedKeys];
|
||||
foreach (string game in games)
|
||||
{
|
||||
// If the game has no items in it, we want to continue
|
||||
var items = GetItemsForBucket(game);
|
||||
if (items == null || items.Count == 0)
|
||||
continue;
|
||||
|
||||
foreach (long id in items.Keys)
|
||||
{
|
||||
var machine = GetMachineForItem(id);
|
||||
if (machine.Value == null)
|
||||
continue;
|
||||
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.CloneOfKey, null);
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.RomOfKey, null);
|
||||
machine.Value.SetFieldValue<string?>(Models.Metadata.Machine.SampleOfKey, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Statistics
|
||||
|
||||
Reference in New Issue
Block a user