claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split Modification class functionality

Browse Source 
This had the potential to cause a lot of issues the way it was. Moving the actual functionality for cleaning, filtering, and applying extras to their appropriate classes allows for less redirection when calling into the code. Modification as a class was essentially a shell around things that should have just been a single call.
This commit is contained in:
Matt Nadareski
2021-02-01 11:43:38 -08:00
parent 99aad96dfb
commit 69010dea7f
15 changed files with 689 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
RombaSharp/Features/Dir2Dat.cs
View File

@@ -64,7 +64,7 @@ namespace RombaSharp.Features
"DatItem.SHA512",
"DatItem.SpamSum",
});
Modification.ApplyCleaning(datfile, cleaner);
cleaner.ApplyCleaning(datfile);
Writer.Write(datfile, outdat);
}
}

36
SabreTools.DatTools/DatFileTool.cs
View File

@@ -14,7 +14,7 @@ namespace SabreTools.DatTools
/// <summary>
/// Helper methods for updating and converting DatFiles
/// </summary>
public class DatFileTool
public static class DatFileTool
{
#region Logging
@@ -25,6 +25,40 @@ namespace SabreTools.DatTools
#endregion
/// <summary>
/// Apply SuperDAT naming logic to a merged DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="inputs">List of inputs to use for renaming</param>
public static void ApplySuperDAT(DatFile datFile, List<ParentablePath> inputs)
{
List<string> keys = datFile.Items.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key].ToList();
List<DatItem> newItems = new List<DatItem>();
foreach (DatItem item in items)
{
DatItem newItem = item;
string filename = inputs[newItem.Source.Index].CurrentPath;
string rootpath = inputs[newItem.Source.Index].ParentPath;
if (!string.IsNullOrWhiteSpace(rootpath))
rootpath += Path.DirectorySeparatorChar.ToString();
filename = filename.Remove(0, rootpath.Length);
newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+ Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+ newItem.Machine.Name;
newItems.Add(newItem);
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, newItems);
});
}
/// <summary>
/// Replace item values from the base set represented by the current DAT
/// </summary>

405
SabreTools.Filtering/Cleaner.cs
View File

@@ -1,6 +1,9 @@
using System.Collections.Generic;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -10,11 +13,13 @@ using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Logging;
[assembly: InternalsVisibleTo("SabreTools.Test")]
namespace SabreTools.Filtering
{
/// <summary>
/// Represents the cleaning operations that need to be performed on a set of items, usually a DAT
/// </summary>
public class Cleaner
{
#region Exclusion Fields
@@ -200,11 +205,91 @@ namespace SabreTools.Filtering
#region Cleaning
/// <summary>
/// Apply cleaning methods to the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if cleaning was successful, false on error</returns>
public bool ApplyCleaning(DatFile datFile, bool throwOnError = false)
{
try
{
// Perform item-level cleaning
CleanDatItems(datFile);
// Bucket and dedupe according to the flag
if (DedupeRoms == DedupeType.Full)
datFile.Items.BucketBy(ItemKey.CRC, DedupeRoms);
else if (DedupeRoms == DedupeType.Game)
datFile.Items.BucketBy(ItemKey.Machine, DedupeRoms);
// Process description to machine name
if (DescriptionAsName == true)
MachineDescriptionToName(datFile);
// If we are removing scene dates, do that now
if (SceneDateStrip == true)
StripSceneDatesFromItems(datFile);
// Run the one rom per game logic, if required
if (OneGamePerRegion == true)
SetOneGamePerRegion(datFile);
// Run the one rom per game logic, if required
if (OneRomPerGame == true)
SetOneRomPerGame(datFile);
// If we are removing fields, do that now
RemoveFieldsFromItems(datFile);
// Remove all marked items
datFile.Items.ClearMarked();
// We remove any blanks, if we aren't supposed to have any
if (KeepEmptyGames == false)
datFile.Items.ClearEmpty();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Clean individual items based on the current filter
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
internal void CleanDatItems(DatFile datFile)
{
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't clean it it
if (item == null)
continue;
// Run cleaning per item
CleanDatItem(item);
}
// Assign back for caution
datFile.Items[key] = items;
}
}
/// <summary>
/// Clean a DatItem according to the cleaner
/// </summary>
/// <param name="datItem">DatItem to clean</param>
public void CleanDatItem(DatItem datItem)
internal void CleanDatItem(DatItem datItem)
{
// If we're stripping unicode characters, strip machine name and description
if (RemoveUnicode)
@@ -243,7 +328,7 @@ namespace SabreTools.Filtering
/// </summary>
/// <param name="game">Name of the game to be cleaned</param>
/// <returns>The cleaned name</returns>
private string CleanGameName(string game)
internal string CleanGameName(string game)
{
if (game == null)
return null;
@@ -258,12 +343,71 @@ namespace SabreTools.Filtering
return game;
}
/// <summary>
/// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
internal void MachineDescriptionToName(DatFile datFile, bool throwOnError = false)
{
try
{
// First we want to get a mapping for all games to description
ConcurrentDictionary<string, string> mapping = new ConcurrentDictionary<string, string>();
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the key mapping doesn't exist, add it
mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -"));
}
});
// Now we loop through every item and update accordingly
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
List<DatItem> newItems = new List<DatItem>();
foreach (DatItem item in items)
{
// Update machine name
if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name))
item.Machine.Name = mapping[item.Machine.Name];
// Update cloneof
if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf))
item.Machine.CloneOf = mapping[item.Machine.CloneOf];
// Update romof
if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf))
item.Machine.RomOf = mapping[item.Machine.RomOf];
// Update sampleof
if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf))
item.Machine.SampleOf = mapping[item.Machine.SampleOf];
// Add the new item to the output list
newItems.Add(item);
}
// Replace the old list of roms with the new one
datFile.Items.Remove(key);
datFile.Items.AddRange(key, newItems);
});
}
catch (Exception ex) when (!throwOnError)
{
logger.Warning(ex.ToString());
}
}
/// <summary>
/// Replace accented characters
/// </summary>
/// <param name="input">String to be parsed</param>
/// <returns>String with characters replaced</returns>
private string NormalizeChars(string input)
internal string NormalizeChars(string input)
{
if (input == null)
return null;
@@ -319,7 +463,7 @@ namespace SabreTools.Filtering
/// </summary>
/// <param name="s">Input string to clean</param>
/// <returns>Cleaned string</returns>
private string RemoveUnicodeCharacters(string s)
internal string RemoveUnicodeCharacters(string s)
{
if (s == null)
return null;
@@ -332,7 +476,7 @@ namespace SabreTools.Filtering
/// </summary>
/// <param name="input">String to be parsed</param>
/// <returns>String with characters replaced</returns>
private string RussianToLatin(string input)
internal string RussianToLatin(string input)
{
if (input == null)
return null;
@@ -367,7 +511,7 @@ namespace SabreTools.Filtering
/// </summary>
/// <param name="input">String to be parsed</param>
/// <returns>String with characters replaced</returns>
private string SearchPattern(string input)
internal string SearchPattern(string input)
{
if (input == null)
return null;
@@ -409,10 +553,238 @@ namespace SabreTools.Filtering
return input;
}
/// <summary>
/// Filter a DAT using 1G1R logic given an ordered set of regions
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <remarks>
/// In the most technical sense, the way that the region list is being used does not
/// confine its values to be just regions. Since it's essentially acting like a
/// specialized version of the machine name filter, anything that is usually encapsulated
/// in parenthesis would be matched on, including disc numbers, languages, editions,
/// and anything else commonly used. Please note that, unlike other existing 1G1R
/// solutions, this does not have the ability to contain custom mappings of parent
/// to clone sets based on name, nor does it have the ability to match on the
/// Release DatItem type.
/// </remarks>
internal void SetOneGamePerRegion(DatFile datFile)
{
// If we have null region list, make it empty
if (RegionList == null)
RegionList = new List<string>();
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Then we want to get a mapping of all machines to parents
Dictionary<string, List<string>> parents = new Dictionary<string, List<string>>();
foreach (string key in datFile.Items.Keys)
{
DatItem item = datFile.Items[key][0];
// Match on CloneOf first
if (!string.IsNullOrEmpty(item.Machine.CloneOf))
{
if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant()))
parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List<string>());
parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Then by RomOf
else if (!string.IsNullOrEmpty(item.Machine.RomOf))
{
if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant()))
parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List<string>());
parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Otherwise, treat it as a parent
else
{
if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant()))
parents.Add(item.Machine.Name.ToLowerInvariant(), new List<string>());
parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
}
// Once we have the full list of mappings, filter out games to keep
foreach (string key in parents.Keys)
{
// Find the first machine that matches the regions in order, if possible
string machine = default;
foreach (string region in RegionList)
{
machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase));
if (machine != default)
break;
}
// If we didn't get a match, use the parent
if (machine == default)
machine = key;
// Remove the key from the list
parents[key].Remove(machine);
// Remove the rest of the items from this key
parents[key].ForEach(k => datFile.Items.Remove(k));
}
// Finally, strip out the parent tags
Splitter.RemoveTagsFromChild(datFile);
}
/// <summary>
/// Ensure that all roms are in their own game (or at least try to ensure)
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
internal void SetOneRomPerGame(DatFile datFile)
{
// Because this introduces subfolders, we need to set the SuperDAT type
datFile.Header.Type = "SuperDAT";
// For each rom, we want to update the game to be "<game name>/<rom name>"
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int i = 0; i < items.Count; i++)
{
DatItemTool.SetOneRomPerGame(items[i]);
}
});
}
/// <summary>
/// Strip the dates from the beginning of scene-style set names
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
internal void StripSceneDatesFromItems(DatFile datFile)
{
// Output the logging statement
logger.User("Stripping scene-style dates");
// Set the regex pattern to use
string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)";
// Now process all of the roms
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int j = 0; j < items.Count; j++)
{
DatItem item = items[j];
if (Regex.IsMatch(item.Machine.Name, pattern))
item.Machine.Name = Regex.Replace(item.Machine.Name, pattern, "$2");
if (Regex.IsMatch(item.Machine.Description, pattern))
item.Machine.Description = Regex.Replace(item.Machine.Description, pattern, "$2");
items[j] = item;
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, items);
});
}
#endregion
#region Filtering
/// <summary>
/// Apply a set of Filters on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="perMachine">True if entire machines are considered, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was filtered, false on error</returns>
public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false)
{
// If we have null filters, return false
if (MachineFilter == null || DatItemFilter == null)
return false;
// If we're filtering per machine, bucket by machine first
if (perMachine)
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
try
{
// Loop over every key in the dictionary
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
bool machinePass = true;
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't pass it
if (item == null)
continue;
// If the item is already filtered out, we skip
if (item.Remove)
continue;
// If the rom doesn't pass the filter, mark for removal
if (!PassesFilters(item))
{
item.Remove = true;
// If we're in machine mode, set and break
if (perMachine)
{
machinePass = false;
break;
}
}
}
// If we didn't pass and we're in machine mode, set all items as remove
if (perMachine && !machinePass)
{
foreach (DatItem item in items)
{
item.Remove = true;
}
}
// Assign back for caution
datFile.Items[key] = items;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Check to see if a DatItem passes the filters
/// </summary>
/// <param name="datItem">DatItem to check</param>
/// <returns>True if the item passed the filter, false otherwise</returns>
internal bool PassesFilters(DatItem datItem)
{
// Null item means it will never pass
if (datItem == null)
return false;
// Filter on Machine fields
if (!MachineFilter.PassesFilters(datItem.Machine))
return false;
// Filter on DatItem fields
return DatItemFilter.PassesFilters(datItem);
}
/// <summary>
/// Split the parts of a filter statement
/// </summary>
@@ -439,25 +811,6 @@ namespace SabreTools.Filtering
return (filterFieldString, filterValue, negate);
}
/// <summary>
/// Check to see if a DatItem passes the filters
/// </summary>
/// <param name="datItem">DatItem to check</param>
/// <returns>True if the item passed the filter, false otherwise</returns>
public bool PassesFilters(DatItem datItem)
{
// Null item means it will never pass
if (datItem == null)
return false;
// Filter on Machine fields
if (!MachineFilter.PassesFilters(datItem.Machine))
return false;
// Filter on DatItem fields
return DatItemFilter.PassesFilters(datItem);
}
#endregion
#region Removal

4
SabreTools.Filtering/DatHeaderRemover.cs
View File

@@ -33,7 +33,7 @@ namespace SabreTools.Filtering
#endregion
#region Remover Population
#region Population
/// <inheritdoc/>
public override bool SetRemover(string field)
@@ -55,7 +55,7 @@ namespace SabreTools.Filtering
#endregion
#region Remover Running
#region Running
/// <summary>
/// Remove fields with given values

4
SabreTools.Filtering/DatItemFilter.cs
View File

@@ -207,7 +207,7 @@ namespace SabreTools.Filtering
#endregion
#region Filter Population
#region Population
/// <summary>
/// Set multiple filters from key
@@ -763,7 +763,7 @@ namespace SabreTools.Filtering
#endregion
#region Filter Running
#region Running
/// <summary>
/// Check to see if a DatItem passes the filters

4
SabreTools.Filtering/DatItemRemover.cs
View File

@@ -38,7 +38,7 @@ namespace SabreTools.Filtering
#endregion
#region Remover Population
#region Population
/// <inheritdoc/>
public override bool SetRemover(string field)
@@ -68,7 +68,7 @@ namespace SabreTools.Filtering
#endregion
#region Remover Running
#region Running
/// <summary>
/// Remove fields with given values

99
SabreTools.Filtering/ExtraIni.cs
View File

@@ -1,6 +1,10 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Logging;
namespace SabreTools.Filtering
@@ -37,7 +41,7 @@ namespace SabreTools.Filtering
#endregion
#region Extras Population
#region Population
/// <summary>
/// Populate item using field:file inputs
@@ -68,5 +72,96 @@ namespace SabreTools.Filtering
}
#endregion
#region Running
/// <summary>
/// Apply a set of Extra INIs on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the extras were applied, false on error</returns>
public bool ApplyExtras(DatFile datFile, bool throwOnError = false)
{
try
{
// Bucket by game first
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
// Create a new set of mappings based on the items
var machineMap = new Dictionary<string, Dictionary<MachineField, string>>();
var datItemMap = new Dictionary<string, Dictionary<DatItemField, string>>();
// Loop through each of the extras
foreach (ExtraIniItem item in Items)
{
foreach (var mapping in item.Mappings)
{
string key = mapping.Key;
List<string> machineNames = mapping.Value;
// Loop through the machines and add the new mappings
foreach (string machine in machineNames)
{
if (item.MachineField != MachineField.NULL)
{
if (!machineMap.ContainsKey(machine))
machineMap[machine] = new Dictionary<MachineField, string>();
machineMap[machine][item.MachineField] = key;
}
else if (item.DatItemField != DatItemField.NULL)
{
if (!datItemMap.ContainsKey(machine))
datItemMap[machine] = new Dictionary<DatItemField, string>();
datItemMap[machine][item.DatItemField] = key;
}
}
}
}
// Now apply the new set of Machine mappings
foreach (string key in machineMap.Keys)
{
// If the key doesn't exist, continue
if (!datFile.Items.ContainsKey(key))
continue;
List<DatItem> datItems = datFile.Items[key];
var mappings = machineMap[key];
foreach (var datItem in datItems)
{
DatItemTool.SetFields(datItem.Machine, mappings);
}
}
// Now apply the new set of DatItem mappings
foreach (string key in datItemMap.Keys)
{
// If the key doesn't exist, continue
if (!datFile.Items.ContainsKey(key))
continue;
List<DatItem> datItems = datFile.Items[key];
var mappings = datItemMap[key];
foreach (var datItem in datItems)
{
DatItemTool.SetFields(datItem, mappings, null);
}
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
#endregion
}
}

4
SabreTools.Filtering/MachineFilter.cs
View File

@@ -112,7 +112,7 @@ namespace SabreTools.Filtering
#endregion
#region Filter Population
#region Population
/// <summary>
/// Set multiple filters from key
@@ -323,7 +323,7 @@ namespace SabreTools.Filtering
#endregion
#region Filter Running
#region Running
/// <summary>
/// Check to see if a Machine passes the filters

634
SabreTools.DatTools/Modification.cs → SabreTools.Filtering/Splitter.cs
View File

@@ -1,25 +1,25 @@
using System;
using System.Collections.Concurrent;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Filtering;
using SabreTools.IO;
using SabreTools.Logging;
namespace SabreTools.DatTools
namespace SabreTools.Filtering
{
public class Splitter
{
#region Fields
/// <summary>
/// Helper methods for cleaning and filtering DatFiles
/// Splitting mode to apply
/// </summary>
public class Modification
{
public MergingFlag SplitType { get; set; }
#endregion
#region Logging
/// <summary>
@@ -29,547 +29,71 @@ namespace SabreTools.DatTools
#endregion
/// <summary>
/// Apply cleaning methods to the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="cleaner">Cleaner to use</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if cleaning was successful, false on error</returns>
public static bool ApplyCleaning(DatFile datFile, Cleaner cleaner, bool throwOnError = false)
{
try
{
// Perform item-level cleaning
CleanDatItems(datFile, cleaner);
// Bucket and dedupe according to the flag
if (cleaner?.DedupeRoms == DedupeType.Full)
datFile.Items.BucketBy(ItemKey.CRC, cleaner.DedupeRoms);
else if (cleaner?.DedupeRoms == DedupeType.Game)
datFile.Items.BucketBy(ItemKey.Machine, cleaner.DedupeRoms);
// Process description to machine name
if (cleaner?.DescriptionAsName == true)
MachineDescriptionToName(datFile);
// If we are removing scene dates, do that now
if (cleaner?.SceneDateStrip == true)
StripSceneDatesFromItems(datFile);
// Run the one rom per game logic, if required
if (cleaner?.OneGamePerRegion == true)
OneGamePerRegion(datFile, cleaner.RegionList);
// Run the one rom per game logic, if required
if (cleaner?.OneRomPerGame == true)
OneRomPerGame(datFile);
// If we are removing fields, do that now
cleaner.RemoveFieldsFromItems(datFile);
// Remove all marked items
datFile.Items.ClearMarked();
// We remove any blanks, if we aren't supposed to have any
if (cleaner?.KeepEmptyGames == false)
datFile.Items.ClearEmpty();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Apply a set of Extra INIs on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="extras">ExtrasIni to use</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the extras were applied, false on error</returns>
public static bool ApplyExtras(DatFile datFile, ExtraIni extras, bool throwOnError = false)
{
try
{
// Bucket by game first
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
// Create a new set of mappings based on the items
var machineMap = new Dictionary<string, Dictionary<MachineField, string>>();
var datItemMap = new Dictionary<string, Dictionary<DatItemField, string>>();
// Loop through each of the extras
foreach (ExtraIniItem item in extras.Items)
{
foreach (var mapping in item.Mappings)
{
string key = mapping.Key;
List<string> machineNames = mapping.Value;
// Loop through the machines and add the new mappings
foreach (string machine in machineNames)
{
if (item.MachineField != MachineField.NULL)
{
if (!machineMap.ContainsKey(machine))
machineMap[machine] = new Dictionary<MachineField, string>();
machineMap[machine][item.MachineField] = key;
}
else if (item.DatItemField != DatItemField.NULL)
{
if (!datItemMap.ContainsKey(machine))
datItemMap[machine] = new Dictionary<DatItemField, string>();
datItemMap[machine][item.DatItemField] = key;
}
}
}
}
// Now apply the new set of Machine mappings
foreach (string key in machineMap.Keys)
{
// If the key doesn't exist, continue
if (!datFile.Items.ContainsKey(key))
continue;
List<DatItem> datItems = datFile.Items[key];
var mappings = machineMap[key];
foreach (var datItem in datItems)
{
DatItemTool.SetFields(datItem.Machine, mappings);
}
}
// Now apply the new set of DatItem mappings
foreach (string key in datItemMap.Keys)
{
// If the key doesn't exist, continue
if (!datFile.Items.ContainsKey(key))
continue;
List<DatItem> datItems = datFile.Items[key];
var mappings = datItemMap[key];
foreach (var datItem in datItems)
{
DatItemTool.SetFields(datItem, mappings, null);
}
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Apply a set of Filters on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="cleaner">Cleaner to use</param>
/// <param name="perMachine">True if entire machines are considered, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was filtered, false on error</returns>
public static bool ApplyFilters(DatFile datFile, Cleaner cleaner, bool perMachine = false, bool throwOnError = false)
{
// If we have a null cleaner or filters, return false
if (cleaner == null || cleaner.MachineFilter == null || cleaner.DatItemFilter == null)
return false;
// If we're filtering per machine, bucket by machine first
if (perMachine)
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
try
{
// Loop over every key in the dictionary
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
bool machinePass = true;
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't pass it
if (item == null)
continue;
// If the item is already filtered out, we skip
if (item.Remove)
continue;
// If the rom doesn't pass the filter, mark for removal
if (!cleaner.PassesFilters(item))
{
item.Remove = true;
// If we're in machine mode, set and break
if (perMachine)
{
machinePass = false;
break;
}
}
}
// If we didn't pass and we're in machine mode, set all items as remove
if (perMachine && !machinePass)
{
foreach (DatItem item in items)
{
item.Remove = true;
}
}
// Assign back for caution
datFile.Items[key] = items;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Apply splitting on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="splitType">Split type to try</param>
/// <param name="useTags">True if DatFile tags override splitting, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was split, false on error</returns>
public static bool ApplySplitting(DatFile datFile, MergingFlag splitType, bool useTags, bool throwOnError = false)
{
try
{
// If we are using tags from the DAT, set the proper input for split type unless overridden
if (useTags && splitType == MergingFlag.None)
splitType = datFile.Header.ForceMerging;
// Run internal splitting
switch (splitType)
{
case MergingFlag.None:
// No-op
break;
case MergingFlag.Device:
CreateDeviceNonMergedSets(datFile, DedupeType.None);
break;
case MergingFlag.Full:
CreateFullyNonMergedSets(datFile, DedupeType.None);
break;
case MergingFlag.NonMerged:
CreateNonMergedSets(datFile, DedupeType.None);
break;
case MergingFlag.Merged:
CreateMergedSets(datFile, DedupeType.None);
break;
case MergingFlag.Split:
CreateSplitSets(datFile, DedupeType.None);
break;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Apply SuperDAT naming logic to a merged DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="inputs">List of inputs to use for renaming</param>
public static void ApplySuperDAT(DatFile datFile, List<ParentablePath> inputs)
{
List<string> keys = datFile.Items.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key].ToList();
List<DatItem> newItems = new List<DatItem>();
foreach (DatItem item in items)
{
DatItem newItem = item;
string filename = inputs[newItem.Source.Index].CurrentPath;
string rootpath = inputs[newItem.Source.Index].ParentPath;
if (!string.IsNullOrWhiteSpace(rootpath))
rootpath += Path.DirectorySeparatorChar.ToString();
filename = filename.Remove(0, rootpath.Length);
newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+ Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+ newItem.Machine.Name;
newItems.Add(newItem);
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, newItems);
});
}
/// <summary>
/// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static void MachineDescriptionToName(DatFile datFile, bool throwOnError = false)
{
try
{
// First we want to get a mapping for all games to description
ConcurrentDictionary<string, string> mapping = new ConcurrentDictionary<string, string>();
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the key mapping doesn't exist, add it
mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -"));
}
});
// Now we loop through every item and update accordingly
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
List<DatItem> newItems = new List<DatItem>();
foreach (DatItem item in items)
{
// Update machine name
if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name))
item.Machine.Name = mapping[item.Machine.Name];
// Update cloneof
if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf))
item.Machine.CloneOf = mapping[item.Machine.CloneOf];
// Update romof
if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf))
item.Machine.RomOf = mapping[item.Machine.RomOf];
// Update sampleof
if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf))
item.Machine.SampleOf = mapping[item.Machine.SampleOf];
// Add the new item to the output list
newItems.Add(item);
}
// Replace the old list of roms with the new one
datFile.Items.Remove(key);
datFile.Items.AddRange(key, newItems);
});
}
catch (Exception ex) when (!throwOnError)
{
logger.Warning(ex.ToString());
}
}
/// <summary>
/// Filter a DAT using 1G1R logic given an ordered set of regions
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="regions">Ordered list of regions to use</param>
/// <remarks>
/// In the most technical sense, the way that the region list is being used does not
/// confine its values to be just regions. Since it's essentially acting like a
/// specialized version of the machine name filter, anything that is usually encapsulated
/// in parenthesis would be matched on, including disc numbers, languages, editions,
/// and anything else commonly used. Please note that, unlike other existing 1G1R
/// solutions, this does not have the ability to contain custom mappings of parent
/// to clone sets based on name, nor does it have the ability to match on the
/// Release DatItem type.
/// </remarks>
public static void OneGamePerRegion(DatFile datFile, List<string> regions)
{
// If we have null region list, make it empty
if (regions == null)
regions = new List<string>();
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Then we want to get a mapping of all machines to parents
Dictionary<string, List<string>> parents = new Dictionary<string, List<string>>();
foreach (string key in datFile.Items.Keys)
{
DatItem item = datFile.Items[key][0];
// Match on CloneOf first
if (!string.IsNullOrEmpty(item.Machine.CloneOf))
{
if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant()))
parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List<string>());
parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Then by RomOf
else if (!string.IsNullOrEmpty(item.Machine.RomOf))
{
if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant()))
parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List<string>());
parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Otherwise, treat it as a parent
else
{
if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant()))
parents.Add(item.Machine.Name.ToLowerInvariant(), new List<string>());
parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
}
// Once we have the full list of mappings, filter out games to keep
foreach (string key in parents.Keys)
{
// Find the first machine that matches the regions in order, if possible
string machine = default;
foreach (string region in regions)
{
machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase));
if (machine != default)
break;
}
// If we didn't get a match, use the parent
if (machine == default)
machine = key;
// Remove the key from the list
parents[key].Remove(machine);
// Remove the rest of the items from this key
parents[key].ForEach(k => datFile.Items.Remove(k));
}
// Finally, strip out the parent tags
RemoveTagsFromChild(datFile);
}
/// <summary>
/// Ensure that all roms are in their own game (or at least try to ensure)
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
public static void OneRomPerGame(DatFile datFile)
{
// Because this introduces subfolders, we need to set the SuperDAT type
datFile.Header.Type = "SuperDAT";
// For each rom, we want to update the game to be "<game name>/<rom name>"
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int i = 0; i < items.Count; i++)
{
DatItemTool.SetOneRomPerGame(items[i]);
}
});
}
/// <summary>
/// Strip the dates from the beginning of scene-style set names
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
public static void StripSceneDatesFromItems(DatFile datFile)
{
// Output the logging statement
logger.User("Stripping scene-style dates");
// Set the regex pattern to use
string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)";
// Now process all of the roms
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int j = 0; j < items.Count; j++)
{
DatItem item = items[j];
if (Regex.IsMatch(item.Machine.Name, pattern))
item.Machine.Name = Regex.Replace(item.Machine.Name, pattern, "$2");
if (Regex.IsMatch(item.Machine.Description, pattern))
item.Machine.Description = Regex.Replace(item.Machine.Description, pattern, "$2");
items[j] = item;
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, items);
});
}
/// <summary>
/// Clean individual items based on the current filter
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="cleaner">Cleaner to use</param>
private static void CleanDatItems(DatFile datFile, Cleaner cleaner)
{
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't clean it it
if (item == null)
continue;
// Run cleaning per item
cleaner.CleanDatItem(item);
}
// Assign back for caution
datFile.Items[key] = items;
}
}
// TODO: Should any of these create a new DatFile in the process?
// The reason this comes up is that doing any of the splits or merges
// is an inherently destructive process. Making it output a new DatFile
// might make it easier to deal with multiple internal steps. On the other
// hand, this will increase memory usage significantly and would force the
// existing paths to behave entirely differently
#region Internal Splitting/Merging
#region Running
/// <summary>
/// Apply splitting on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="useTags">True if DatFile tags override splitting, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was split, false on error</returns>
public bool ApplySplitting(DatFile datFile, bool useTags, bool throwOnError = false)
{
try
{
// If we are using tags from the DAT, set the proper input for split type unless overridden
if (useTags && SplitType == MergingFlag.None)
SplitType = datFile.Header.ForceMerging;
// Run internal splitting
switch (SplitType)
{
case MergingFlag.None:
// No-op
break;
case MergingFlag.Device:
CreateDeviceNonMergedSets(datFile);
break;
case MergingFlag.Full:
CreateFullyNonMergedSets(datFile);
break;
case MergingFlag.NonMerged:
CreateNonMergedSets(datFile);
break;
case MergingFlag.Merged:
CreateMergedSets(datFile);
break;
case MergingFlag.Split:
CreateSplitSets(datFile);
break;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Use cdevice_ref tags to get full non-merged sets and remove parenting tags
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="mergeroms">Dedupe type to be used</param>
private static void CreateDeviceNonMergedSets(DatFile datFile, DedupeType mergeroms)
internal static void CreateDeviceNonMergedSets(DatFile datFile)
{
logger.User("Creating device non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true);
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Now we want to loop through all of the games and set the correct information
while (AddRomsFromDevices(datFile, false, false)) ;
@@ -583,13 +107,12 @@ namespace SabreTools.DatTools
/// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="mergeroms">Dedupe type to be used</param>
private static void CreateFullyNonMergedSets(DatFile datFile, DedupeType mergeroms)
internal static void CreateFullyNonMergedSets(DatFile datFile)
{
logger.User("Creating fully non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true);
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Now we want to loop through all of the games and set the correct information
while (AddRomsFromDevices(datFile, true, true)) ;
@@ -607,13 +130,12 @@ namespace SabreTools.DatTools
/// Use cloneof tags to create merged sets and remove the tags
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="mergeroms">Dedupe type to be used</param>
private static void CreateMergedSets(DatFile datFile, DedupeType mergeroms)
internal static void CreateMergedSets(DatFile datFile)
{
logger.User("Creating merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true);
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Now we want to loop through all of the games and set the correct information
AddRomsFromChildren(datFile);
@@ -630,13 +152,12 @@ namespace SabreTools.DatTools
/// Use cloneof tags to create non-merged sets and remove the tags
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="mergeroms">Dedupe type to be used</param>
private static void CreateNonMergedSets(DatFile datFile, DedupeType mergeroms)
internal static void CreateNonMergedSets(DatFile datFile)
{
logger.User("Creating non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true);
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Now we want to loop through all of the games and set the correct information
AddRomsFromParent(datFile);
@@ -653,13 +174,12 @@ namespace SabreTools.DatTools
/// Use cloneof and romof tags to create split sets and remove the tags
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="mergeroms">Dedupe type to be used</param>
private static void CreateSplitSets(DatFile datFile, DedupeType mergeroms)
internal static void CreateSplitSets(DatFile datFile)
{
logger.User("Creating split sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true);
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Now we want to loop through all of the games and set the correct information
RemoveRomsFromChild(datFile);
@@ -676,7 +196,7 @@ namespace SabreTools.DatTools
/// Use romof tags to add roms to the children
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
private static void AddRomsFromBios(DatFile datFile)
internal static void AddRomsFromBios(DatFile datFile)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
@@ -717,7 +237,7 @@ namespace SabreTools.DatTools
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="dev">True if only child device sets are touched, false for non-device sets (default)</param>
/// <param name="useSlotOptions">True if slotoptions tags are used as well, false otherwise</param>
private static bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool useSlotOptions = false)
internal static bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool useSlotOptions = false)
{
bool foundnew = false;
List<string> machines = datFile.Items.Keys.OrderBy(g => g).ToList();
@@ -845,7 +365,7 @@ namespace SabreTools.DatTools
/// Use cloneof tags to add roms to the children, setting the new romof tag in the process
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
private static void AddRomsFromParent(DatFile datFile)
internal static void AddRomsFromParent(DatFile datFile)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
@@ -896,7 +416,7 @@ namespace SabreTools.DatTools
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="subfolder">True to add DatItems to subfolder of parent (not including Disk), false otherwise</param>
private static void AddRomsFromChildren(DatFile datFile, bool subfolder = true)
internal static void AddRomsFromChildren(DatFile datFile, bool subfolder = true)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
@@ -1008,7 +528,7 @@ namespace SabreTools.DatTools
/// Remove all BIOS and device sets
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
private static void RemoveBiosAndDeviceSets(DatFile datFile)
internal static void RemoveBiosAndDeviceSets(DatFile datFile)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
@@ -1027,7 +547,7 @@ namespace SabreTools.DatTools
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="bios">True if only child Bios sets are touched, false for non-bios sets (default)</param>
private static void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false)
internal static void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false)
{
// Loop through the romof tags
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
@@ -1071,7 +591,7 @@ namespace SabreTools.DatTools
/// Use cloneof tags to remove roms from the children
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
private static void RemoveRomsFromChild(DatFile datFile)
internal static void RemoveRomsFromChild(DatFile datFile)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
@@ -1118,7 +638,7 @@ namespace SabreTools.DatTools
/// Remove all romof and cloneof tags from all games
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
private static void RemoveTagsFromChild(DatFile datFile)
internal static void RemoveTagsFromChild(DatFile datFile)
{
List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)

58
SabreTools/Features/BaseFeature.cs
View File

@@ -1735,6 +1735,11 @@ Some special strings that can be used:
/// </summary>
protected string OutputDir { get; set; }
/// <summary>
/// Pre-configured Splitter
/// </summary>
protected Filtering.Splitter Splitter { get; set; }
#endregion
#region Add Feature Groups
@@ -1801,6 +1806,7 @@ Some special strings that can be used:
Extras = GetExtras(features);
Header = GetDatHeader(features);
OutputDir = GetString(features, OutputDirStringValue);
Splitter = GetSplitter(features);
// Set threading flag, if necessary
if (features.ContainsKey(ThreadsInt32Value))
@@ -1895,26 +1901,6 @@ Some special strings that can be used:
return splittingMode;
}
/// <summary>
/// Get SplitType from feature list
/// </summary>
protected MergingFlag GetSplitType(Dictionary<string, Feature> features)
{
MergingFlag splitType = MergingFlag.None;
if (GetBoolean(features, DatDeviceNonMergedValue))
splitType = MergingFlag.Device;
else if (GetBoolean(features, DatFullNonMergedValue))
splitType = MergingFlag.Full;
else if (GetBoolean(features, DatMergedValue))
splitType = MergingFlag.Merged;
else if (GetBoolean(features, DatNonMergedValue))
splitType = MergingFlag.NonMerged;
else if (GetBoolean(features, DatSplitValue))
splitType = MergingFlag.Split;
return splitType;
}
/// <summary>
/// Get StatReportFormat from feature list
/// </summary>
@@ -2135,6 +2121,38 @@ Some special strings that can be used:
return extraIni;
}
/// <summary>
/// Get Splitter from feature list
/// </summary>
private Filtering.Splitter GetSplitter(Dictionary<string, Feature> features)
{
Filtering.Splitter splitter = new Filtering.Splitter
{
SplitType = GetSplitType(features),
};
return splitter;
}
/// <summary>
/// Get SplitType from feature list
/// </summary>
private MergingFlag GetSplitType(Dictionary<string, Feature> features)
{
MergingFlag splitType = MergingFlag.None;
if (GetBoolean(features, DatDeviceNonMergedValue))
splitType = MergingFlag.Device;
else if (GetBoolean(features, DatFullNonMergedValue))
splitType = MergingFlag.Full;
else if (GetBoolean(features, DatMergedValue))
splitType = MergingFlag.Merged;
else if (GetBoolean(features, DatNonMergedValue))
splitType = MergingFlag.NonMerged;
else if (GetBoolean(features, DatSplitValue))
splitType = MergingFlag.Split;
return splitType;
}
#endregion
#region Protected Helpers

27
SabreTools/Features/Batch.cs
View File

@@ -168,7 +168,7 @@ Reset the internal state: reset();";
"DatItem.SpamSum",
"DatItem.Date",
});
Modification.ApplyCleaning(datFile, dfdCleaner);
dfdCleaner.ApplyCleaning(datFile);
break;
@@ -213,18 +213,18 @@ Reset the internal state: reset();";
}
// Create cleaner to run filters from
Cleaner cleaner = new Cleaner
Cleaner filterCleaner = new Cleaner
{
MachineFilter = new MachineFilter(),
DatItemFilter = new DatItemFilter(),
};
// Set the possible filters
cleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value);
cleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value);
filterCleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value);
filterCleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value);
// Apply the filters blindly
Modification.ApplyFilters(datFile, cleaner, filterPerMachine.Value);
filterCleaner.ApplyFilters(datFile, filterPerMachine.Value);
// Cleanup after the filter
// TODO: We might not want to remove immediately
@@ -269,7 +269,7 @@ Reset the internal state: reset();";
extraIni.Items.Add(extraIniItem);
// Apply the extra INI blindly
Modification.ApplyExtras(datFile, extraIni);
extraIni.ApplyExtras(datFile);
break;
@@ -293,7 +293,8 @@ Reset the internal state: reset();";
}
// Apply the merging flag
Modification.ApplySplitting(datFile, mergingFlag, false);
Filtering.Splitter splitter = new Filtering.Splitter { SplitType = mergingFlag };
splitter.ApplySplitting(datFile, false);
break;
@@ -307,7 +308,8 @@ Reset the internal state: reset();";
}
// Apply the logic
Modification.MachineDescriptionToName(datFile);
Cleaner descNameCleaner = new Cleaner { DescriptionAsName = true };
descNameCleaner.ApplyCleaning(datFile);
break;
@@ -321,7 +323,8 @@ Reset the internal state: reset();";
}
// Run the 1G1R functionality
Modification.OneGamePerRegion(datFile, command.Arguments);
Cleaner ogorCleaner = new Cleaner { OneGamePerRegion = true, RegionList = command.Arguments };
ogorCleaner.ApplyCleaning(datFile);
break;
@@ -335,7 +338,8 @@ Reset the internal state: reset();";
}
// Apply the logic
Modification.OneRomPerGame(datFile);
Cleaner orpgCleaner = new Cleaner { OneRomPerGame = true };
orpgCleaner.ApplyCleaning(datFile);
break;
@@ -365,7 +369,8 @@ Reset the internal state: reset();";
}
// Apply the logic
Modification.StripSceneDatesFromItems(datFile);
Cleaner stripCleaner = new Cleaner { SceneDateStrip = true };
stripCleaner.ApplyCleaning(datFile);
break;

11
SabreTools/Features/DatFromDir.cs
View File

@@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.FileTypes;
@@ -62,9 +61,9 @@ namespace SabreTools.Features
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
var splitType = GetSplitType(features);
// Apply the specialized field removals to the cleaner
if (!addFileDates)
Cleaner.PopulateExclusionsFromList(new List<string> { "DatItem.Date" });
// Create a new DATFromDir object and process the inputs
@@ -95,10 +94,10 @@ namespace SabreTools.Features
if (success)
{
// Perform additional processing steps
Modification.ApplyExtras(datdata, Extras);
Modification.ApplySplitting(datdata, splitType, false);
Modification.ApplyFilters(datdata, Cleaner);
Modification.ApplyCleaning(datdata, Cleaner);
Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, false);
Cleaner.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata);
// Write out the file
Writer.Write(datdata, OutputDir);

10
SabreTools/Features/Split.cs
View File

@@ -65,7 +65,7 @@ namespace SabreTools.Features
// Extension splitting
if (splittingMode.HasFlag(SplittingMode.Extension))
{
(DatFile extADat, DatFile extBDat) = Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue));
(DatFile extADat, DatFile extBDat) = DatTools.Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue));
InternalStopwatch watch = new InternalStopwatch("Outputting extension-split DATs");
@@ -79,7 +79,7 @@ namespace SabreTools.Features
// Hash splitting
if (splittingMode.HasFlag(SplittingMode.Hash))
{
Dictionary<DatItemField, DatFile> typeDats = Splitter.SplitByHash(internalDat);
Dictionary<DatItemField, DatFile> typeDats = DatTools.Splitter.SplitByHash(internalDat);
InternalStopwatch watch = new InternalStopwatch("Outputting hash-split DATs");
@@ -96,7 +96,7 @@ namespace SabreTools.Features
if (splittingMode.HasFlag(SplittingMode.Level))
{
logger.Warning("This feature is not implemented: level-split");
Splitter.SplitByLevel(
DatTools.Splitter.SplitByLevel(
internalDat,
OutputDir,
GetBoolean(features, ShortValue),
@@ -106,7 +106,7 @@ namespace SabreTools.Features
// Size splitting
if (splittingMode.HasFlag(SplittingMode.Size))
{
(DatFile lessThan, DatFile greaterThan) = Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value));
(DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value));
InternalStopwatch watch = new InternalStopwatch("Outputting size-split DATs");
@@ -120,7 +120,7 @@ namespace SabreTools.Features
// Type splitting
if (splittingMode.HasFlag(SplittingMode.Type))
{
Dictionary<ItemType, DatFile> typeDats = Splitter.SplitByType(internalDat);
Dictionary<ItemType, DatFile> typeDats = DatTools.Splitter.SplitByType(internalDat);
InternalStopwatch watch = new InternalStopwatch("Outputting ItemType DATs");

34
SabreTools/Features/Update.cs
View File

@@ -156,10 +156,10 @@ namespace SabreTools.Features
|| datFile.Header.DatFormat.HasFlag(DatFormat.SSV));
// Perform additional processing steps
Modification.ApplyExtras(datFile, Extras);
Modification.ApplySplitting(datFile, GetSplitType(features), false);
Modification.ApplyFilters(datFile, Cleaner);
Modification.ApplyCleaning(datFile, Cleaner);
Extras.ApplyExtras(datFile);
Splitter.ApplySplitting(datFile, false);
Cleaner.ApplyFilters(datFile);
Cleaner.ApplyCleaning(datFile);
// Get the correct output path
string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
@@ -194,10 +194,10 @@ namespace SabreTools.Features
datHeaders = DatFileTool.PopulateUserData(userInputDat, inputPaths);
// Perform additional processing steps
Modification.ApplyExtras(userInputDat, Extras);
Modification.ApplySplitting(userInputDat, GetSplitType(features), false);
Modification.ApplyFilters(userInputDat, Cleaner);
Modification.ApplyCleaning(userInputDat, Cleaner);
Extras.ApplyExtras(userInputDat);
Splitter.ApplySplitting(userInputDat, false);
Cleaner.ApplyFilters(userInputDat);
Cleaner.ApplyCleaning(userInputDat);
// Output only DatItems that are duplicated across inputs
if (updateMode.HasFlag(UpdateMode.DiffDupesOnly))
@@ -286,10 +286,10 @@ namespace SabreTools.Features
Parser.ParseInto(repDat, inputPath, indexId: 1, keep: true);
// Perform additional processing steps
Modification.ApplyExtras(repDat, Extras);
Modification.ApplySplitting(repDat, GetSplitType(features), false);
Modification.ApplyFilters(repDat, Cleaner);
Modification.ApplyCleaning(repDat, Cleaner);
Extras.ApplyExtras(repDat);
Splitter.ApplySplitting(repDat, false);
Cleaner.ApplyFilters(repDat);
Cleaner.ApplyCleaning(repDat);
// Now replace the fields from the base DatFile
DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue));
@@ -311,10 +311,10 @@ namespace SabreTools.Features
Parser.ParseInto(repDat, inputPath, indexId: 1, keep: true);
// Perform additional processing steps
Modification.ApplyExtras(repDat, Extras);
Modification.ApplySplitting(repDat, GetSplitType(features), false);
Modification.ApplyFilters(repDat, Cleaner);
Modification.ApplyCleaning(repDat, Cleaner);
Extras.ApplyExtras(repDat);
Splitter.ApplySplitting(repDat, false);
Cleaner.ApplyFilters(repDat);
Cleaner.ApplyCleaning(repDat);
// Now replace the fields from the base DatFile
DatFileTool.BaseReplace(
@@ -336,7 +336,7 @@ namespace SabreTools.Features
{
// If we're in SuperDAT mode, prefix all games with their respective DATs
if (string.Equals(userInputDat.Header.Type, "SuperDAT", StringComparison.OrdinalIgnoreCase))
Modification.ApplySuperDAT(userInputDat, inputPaths);
DatFileTool.ApplySuperDAT(userInputDat, inputPaths);
Writer.Write(userInputDat, OutputDir);
}

17
SabreTools/Features/Verify.cs
View File

@@ -50,7 +50,6 @@ namespace SabreTools.Features
TreatAsFile asFiles = GetTreatAsFiles(features);
bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue);
var splitType = GetSplitType(features);
// If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue))
@@ -62,10 +61,10 @@ namespace SabreTools.Features
Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true);
// Perform additional processing steps
Modification.ApplyExtras(datdata, Extras);
Modification.ApplySplitting(datdata, splitType, true);
Modification.ApplyFilters(datdata, Cleaner);
Modification.ApplyCleaning(datdata, Cleaner);
Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, true);
Cleaner.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
@@ -109,10 +108,10 @@ namespace SabreTools.Features
}
// Perform additional processing steps
Modification.ApplyExtras(datdata, Extras);
Modification.ApplySplitting(datdata, splitType, true);
Modification.ApplyFilters(datdata, Cleaner);
Modification.ApplyCleaning(datdata, Cleaner);
Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, true);
Cleaner.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;