From 69010dea7f71b5af428479e20c33b7d1abac1c08 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Mon, 1 Feb 2021 11:43:38 -0800 Subject: [PATCH] Split Modification class functionality This had the potential to cause a lot of issues the way it was. Moving the actual functionality for cleaning, filtering, and applying extras to their appropriate classes allows for less redirection when calling into the code. Modification as a class was essentially a shell around things that should have just been a single call. --- RombaSharp/Features/Dir2Dat.cs | 2 +- SabreTools.DatTools/DatFileTool.cs | 36 +- SabreTools.Filtering/Cleaner.cs | 405 ++++++++++- SabreTools.Filtering/DatHeaderRemover.cs | 4 +- SabreTools.Filtering/DatItemFilter.cs | 4 +- SabreTools.Filtering/DatItemRemover.cs | 4 +- SabreTools.Filtering/ExtraIni.cs | 99 ++- SabreTools.Filtering/MachineFilter.cs | 4 +- .../Splitter.cs | 638 +++--------------- SabreTools/Features/BaseFeature.cs | 58 +- SabreTools/Features/Batch.cs | 27 +- SabreTools/Features/DatFromDir.cs | 13 +- SabreTools/Features/Split.cs | 10 +- SabreTools/Features/Update.cs | 34 +- SabreTools/Features/Verify.cs | 17 +- 15 files changed, 689 insertions(+), 666 deletions(-) rename SabreTools.DatTools/Modification.cs => SabreTools.Filtering/Splitter.cs (54%) diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index a3469500..7836a4b1 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -64,7 +64,7 @@ namespace RombaSharp.Features "DatItem.SHA512", "DatItem.SpamSum", }); - Modification.ApplyCleaning(datfile, cleaner); + cleaner.ApplyCleaning(datfile); Writer.Write(datfile, outdat); } } diff --git a/SabreTools.DatTools/DatFileTool.cs b/SabreTools.DatTools/DatFileTool.cs index 4b40dccb..5fc48dfa 100644 --- a/SabreTools.DatTools/DatFileTool.cs +++ b/SabreTools.DatTools/DatFileTool.cs @@ -14,7 +14,7 @@ namespace SabreTools.DatTools /// /// Helper methods for updating and converting DatFiles /// - public class DatFileTool + public static class DatFileTool { #region Logging @@ -25,6 +25,40 @@ namespace SabreTools.DatTools #endregion + /// + /// Apply SuperDAT naming logic to a merged DatFile + /// + /// Current DatFile object to run operations on + /// List of inputs to use for renaming + public static void ApplySuperDAT(DatFile datFile, List inputs) + { + List keys = datFile.Items.Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key].ToList(); + List newItems = new List(); + foreach (DatItem item in items) + { + DatItem newItem = item; + string filename = inputs[newItem.Source.Index].CurrentPath; + string rootpath = inputs[newItem.Source.Index].ParentPath; + + if (!string.IsNullOrWhiteSpace(rootpath)) + rootpath += Path.DirectorySeparatorChar.ToString(); + + filename = filename.Remove(0, rootpath.Length); + newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar + + Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar + + newItem.Machine.Name; + + newItems.Add(newItem); + } + + datFile.Items.Remove(key); + datFile.Items.AddRange(key, newItems); + }); + } + /// /// Replace item values from the base set represented by the current DAT /// diff --git a/SabreTools.Filtering/Cleaner.cs b/SabreTools.Filtering/Cleaner.cs index 7d5ba136..b1635780 100644 --- a/SabreTools.Filtering/Cleaner.cs +++ b/SabreTools.Filtering/Cleaner.cs @@ -1,6 +1,9 @@ -using System.Collections.Generic; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -10,11 +13,13 @@ using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.Logging; +[assembly: InternalsVisibleTo("SabreTools.Test")] namespace SabreTools.Filtering { /// /// Represents the cleaning operations that need to be performed on a set of items, usually a DAT /// + public class Cleaner { #region Exclusion Fields @@ -200,11 +205,91 @@ namespace SabreTools.Filtering #region Cleaning + /// + /// Apply cleaning methods to the DatFile + /// + /// Current DatFile object to run operations on + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if cleaning was successful, false on error + public bool ApplyCleaning(DatFile datFile, bool throwOnError = false) + { + try + { + // Perform item-level cleaning + CleanDatItems(datFile); + + // Bucket and dedupe according to the flag + if (DedupeRoms == DedupeType.Full) + datFile.Items.BucketBy(ItemKey.CRC, DedupeRoms); + else if (DedupeRoms == DedupeType.Game) + datFile.Items.BucketBy(ItemKey.Machine, DedupeRoms); + + // Process description to machine name + if (DescriptionAsName == true) + MachineDescriptionToName(datFile); + + // If we are removing scene dates, do that now + if (SceneDateStrip == true) + StripSceneDatesFromItems(datFile); + + // Run the one rom per game logic, if required + if (OneGamePerRegion == true) + SetOneGamePerRegion(datFile); + + // Run the one rom per game logic, if required + if (OneRomPerGame == true) + SetOneRomPerGame(datFile); + + // If we are removing fields, do that now + RemoveFieldsFromItems(datFile); + + // Remove all marked items + datFile.Items.ClearMarked(); + + // We remove any blanks, if we aren't supposed to have any + if (KeepEmptyGames == false) + datFile.Items.ClearEmpty(); + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + /// + /// Clean individual items based on the current filter + /// + /// Current DatFile object to run operations on + internal void CleanDatItems(DatFile datFile) + { + List keys = datFile.Items.Keys.ToList(); + foreach (string key in keys) + { + // For every item in the current key + List items = datFile.Items[key]; + foreach (DatItem item in items) + { + // If we have a null item, we can't clean it it + if (item == null) + continue; + + // Run cleaning per item + CleanDatItem(item); + } + + // Assign back for caution + datFile.Items[key] = items; + } + } + /// /// Clean a DatItem according to the cleaner /// /// DatItem to clean - public void CleanDatItem(DatItem datItem) + internal void CleanDatItem(DatItem datItem) { // If we're stripping unicode characters, strip machine name and description if (RemoveUnicode) @@ -243,7 +328,7 @@ namespace SabreTools.Filtering /// /// Name of the game to be cleaned /// The cleaned name - private string CleanGameName(string game) + internal string CleanGameName(string game) { if (game == null) return null; @@ -258,12 +343,71 @@ namespace SabreTools.Filtering return game; } + /// + /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof + /// + /// Current DatFile object to run operations on + /// True if the error that is thrown should be thrown back to the caller, false otherwise + internal void MachineDescriptionToName(DatFile datFile, bool throwOnError = false) + { + try + { + // First we want to get a mapping for all games to description + ConcurrentDictionary mapping = new ConcurrentDictionary(); + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + foreach (DatItem item in items) + { + // If the key mapping doesn't exist, add it + mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); + } + }); + + // Now we loop through every item and update accordingly + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + List newItems = new List(); + foreach (DatItem item in items) + { + // Update machine name + if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name)) + item.Machine.Name = mapping[item.Machine.Name]; + + // Update cloneof + if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf)) + item.Machine.CloneOf = mapping[item.Machine.CloneOf]; + + // Update romof + if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf)) + item.Machine.RomOf = mapping[item.Machine.RomOf]; + + // Update sampleof + if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf)) + item.Machine.SampleOf = mapping[item.Machine.SampleOf]; + + // Add the new item to the output list + newItems.Add(item); + } + + // Replace the old list of roms with the new one + datFile.Items.Remove(key); + datFile.Items.AddRange(key, newItems); + }); + } + catch (Exception ex) when (!throwOnError) + { + logger.Warning(ex.ToString()); + } + } + /// /// Replace accented characters /// /// String to be parsed /// String with characters replaced - private string NormalizeChars(string input) + internal string NormalizeChars(string input) { if (input == null) return null; @@ -319,7 +463,7 @@ namespace SabreTools.Filtering /// /// Input string to clean /// Cleaned string - private string RemoveUnicodeCharacters(string s) + internal string RemoveUnicodeCharacters(string s) { if (s == null) return null; @@ -332,7 +476,7 @@ namespace SabreTools.Filtering /// /// String to be parsed /// String with characters replaced - private string RussianToLatin(string input) + internal string RussianToLatin(string input) { if (input == null) return null; @@ -367,7 +511,7 @@ namespace SabreTools.Filtering /// /// String to be parsed /// String with characters replaced - private string SearchPattern(string input) + internal string SearchPattern(string input) { if (input == null) return null; @@ -409,10 +553,238 @@ namespace SabreTools.Filtering return input; } + /// + /// Filter a DAT using 1G1R logic given an ordered set of regions + /// + /// Current DatFile object to run operations on + /// + /// In the most technical sense, the way that the region list is being used does not + /// confine its values to be just regions. Since it's essentially acting like a + /// specialized version of the machine name filter, anything that is usually encapsulated + /// in parenthesis would be matched on, including disc numbers, languages, editions, + /// and anything else commonly used. Please note that, unlike other existing 1G1R + /// solutions, this does not have the ability to contain custom mappings of parent + /// to clone sets based on name, nor does it have the ability to match on the + /// Release DatItem type. + /// + internal void SetOneGamePerRegion(DatFile datFile) + { + // If we have null region list, make it empty + if (RegionList == null) + RegionList = new List(); + + // For sake of ease, the first thing we want to do is bucket by game + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); + + // Then we want to get a mapping of all machines to parents + Dictionary> parents = new Dictionary>(); + foreach (string key in datFile.Items.Keys) + { + DatItem item = datFile.Items[key][0]; + + // Match on CloneOf first + if (!string.IsNullOrEmpty(item.Machine.CloneOf)) + { + if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant())) + parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List()); + + parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); + } + + // Then by RomOf + else if (!string.IsNullOrEmpty(item.Machine.RomOf)) + { + if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant())) + parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List()); + + parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); + } + + // Otherwise, treat it as a parent + else + { + if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant())) + parents.Add(item.Machine.Name.ToLowerInvariant(), new List()); + + parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); + } + } + + // Once we have the full list of mappings, filter out games to keep + foreach (string key in parents.Keys) + { + // Find the first machine that matches the regions in order, if possible + string machine = default; + foreach (string region in RegionList) + { + machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase)); + if (machine != default) + break; + } + + // If we didn't get a match, use the parent + if (machine == default) + machine = key; + + // Remove the key from the list + parents[key].Remove(machine); + + // Remove the rest of the items from this key + parents[key].ForEach(k => datFile.Items.Remove(k)); + } + + // Finally, strip out the parent tags + Splitter.RemoveTagsFromChild(datFile); + } + + /// + /// Ensure that all roms are in their own game (or at least try to ensure) + /// + /// Current DatFile object to run operations on + internal void SetOneRomPerGame(DatFile datFile) + { + // Because this introduces subfolders, we need to set the SuperDAT type + datFile.Header.Type = "SuperDAT"; + + // For each rom, we want to update the game to be "/" + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + for (int i = 0; i < items.Count; i++) + { + DatItemTool.SetOneRomPerGame(items[i]); + } + }); + } + + /// + /// Strip the dates from the beginning of scene-style set names + /// + /// Current DatFile object to run operations on + internal void StripSceneDatesFromItems(DatFile datFile) + { + // Output the logging statement + logger.User("Stripping scene-style dates"); + + // Set the regex pattern to use + string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)"; + + // Now process all of the roms + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + for (int j = 0; j < items.Count; j++) + { + DatItem item = items[j]; + if (Regex.IsMatch(item.Machine.Name, pattern)) + item.Machine.Name = Regex.Replace(item.Machine.Name, pattern, "$2"); + + if (Regex.IsMatch(item.Machine.Description, pattern)) + item.Machine.Description = Regex.Replace(item.Machine.Description, pattern, "$2"); + + items[j] = item; + } + + datFile.Items.Remove(key); + datFile.Items.AddRange(key, items); + }); + } + #endregion #region Filtering + /// + /// Apply a set of Filters on the DatFile + /// + /// Current DatFile object to run operations on + /// True if entire machines are considered, false otherwise (default) + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if the DatFile was filtered, false on error + public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false) + { + // If we have null filters, return false + if (MachineFilter == null || DatItemFilter == null) + return false; + + // If we're filtering per machine, bucket by machine first + if (perMachine) + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); + + try + { + // Loop over every key in the dictionary + List keys = datFile.Items.Keys.ToList(); + foreach (string key in keys) + { + // For every item in the current key + bool machinePass = true; + List items = datFile.Items[key]; + foreach (DatItem item in items) + { + // If we have a null item, we can't pass it + if (item == null) + continue; + + // If the item is already filtered out, we skip + if (item.Remove) + continue; + + // If the rom doesn't pass the filter, mark for removal + if (!PassesFilters(item)) + { + item.Remove = true; + + // If we're in machine mode, set and break + if (perMachine) + { + machinePass = false; + break; + } + } + } + + // If we didn't pass and we're in machine mode, set all items as remove + if (perMachine && !machinePass) + { + foreach (DatItem item in items) + { + item.Remove = true; + } + } + + // Assign back for caution + datFile.Items[key] = items; + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + /// + /// Check to see if a DatItem passes the filters + /// + /// DatItem to check + /// True if the item passed the filter, false otherwise + internal bool PassesFilters(DatItem datItem) + { + // Null item means it will never pass + if (datItem == null) + return false; + + // Filter on Machine fields + if (!MachineFilter.PassesFilters(datItem.Machine)) + return false; + + // Filter on DatItem fields + return DatItemFilter.PassesFilters(datItem); + } + /// /// Split the parts of a filter statement /// @@ -439,25 +811,6 @@ namespace SabreTools.Filtering return (filterFieldString, filterValue, negate); } - /// - /// Check to see if a DatItem passes the filters - /// - /// DatItem to check - /// True if the item passed the filter, false otherwise - public bool PassesFilters(DatItem datItem) - { - // Null item means it will never pass - if (datItem == null) - return false; - - // Filter on Machine fields - if (!MachineFilter.PassesFilters(datItem.Machine)) - return false; - - // Filter on DatItem fields - return DatItemFilter.PassesFilters(datItem); - } - #endregion #region Removal diff --git a/SabreTools.Filtering/DatHeaderRemover.cs b/SabreTools.Filtering/DatHeaderRemover.cs index 12bbb60a..18bf1a02 100644 --- a/SabreTools.Filtering/DatHeaderRemover.cs +++ b/SabreTools.Filtering/DatHeaderRemover.cs @@ -33,7 +33,7 @@ namespace SabreTools.Filtering #endregion - #region Remover Population + #region Population /// public override bool SetRemover(string field) @@ -55,7 +55,7 @@ namespace SabreTools.Filtering #endregion - #region Remover Running + #region Running /// /// Remove fields with given values diff --git a/SabreTools.Filtering/DatItemFilter.cs b/SabreTools.Filtering/DatItemFilter.cs index 47023f49..db2793e6 100644 --- a/SabreTools.Filtering/DatItemFilter.cs +++ b/SabreTools.Filtering/DatItemFilter.cs @@ -207,7 +207,7 @@ namespace SabreTools.Filtering #endregion - #region Filter Population + #region Population /// /// Set multiple filters from key @@ -763,7 +763,7 @@ namespace SabreTools.Filtering #endregion - #region Filter Running + #region Running /// /// Check to see if a DatItem passes the filters diff --git a/SabreTools.Filtering/DatItemRemover.cs b/SabreTools.Filtering/DatItemRemover.cs index e688f69b..c6341093 100644 --- a/SabreTools.Filtering/DatItemRemover.cs +++ b/SabreTools.Filtering/DatItemRemover.cs @@ -38,7 +38,7 @@ namespace SabreTools.Filtering #endregion - #region Remover Population + #region Population /// public override bool SetRemover(string field) @@ -68,7 +68,7 @@ namespace SabreTools.Filtering #endregion - #region Remover Running + #region Running /// /// Remove fields with given values diff --git a/SabreTools.Filtering/ExtraIni.cs b/SabreTools.Filtering/ExtraIni.cs index cb9653b8..575a8482 100644 --- a/SabreTools.Filtering/ExtraIni.cs +++ b/SabreTools.Filtering/ExtraIni.cs @@ -1,6 +1,10 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using SabreTools.Core; using SabreTools.Core.Tools; +using SabreTools.DatFiles; +using SabreTools.DatItems; using SabreTools.Logging; namespace SabreTools.Filtering @@ -37,7 +41,7 @@ namespace SabreTools.Filtering #endregion - #region Extras Population + #region Population /// /// Populate item using field:file inputs @@ -68,5 +72,96 @@ namespace SabreTools.Filtering } #endregion + + #region Running + + /// + /// Apply a set of Extra INIs on the DatFile + /// + /// Current DatFile object to run operations on + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if the extras were applied, false on error + public bool ApplyExtras(DatFile datFile, bool throwOnError = false) + { + try + { + // Bucket by game first + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); + + // Create a new set of mappings based on the items + var machineMap = new Dictionary>(); + var datItemMap = new Dictionary>(); + + // Loop through each of the extras + foreach (ExtraIniItem item in Items) + { + foreach (var mapping in item.Mappings) + { + string key = mapping.Key; + List machineNames = mapping.Value; + + // Loop through the machines and add the new mappings + foreach (string machine in machineNames) + { + if (item.MachineField != MachineField.NULL) + { + if (!machineMap.ContainsKey(machine)) + machineMap[machine] = new Dictionary(); + + machineMap[machine][item.MachineField] = key; + } + else if (item.DatItemField != DatItemField.NULL) + { + if (!datItemMap.ContainsKey(machine)) + datItemMap[machine] = new Dictionary(); + + datItemMap[machine][item.DatItemField] = key; + } + } + } + } + + // Now apply the new set of Machine mappings + foreach (string key in machineMap.Keys) + { + // If the key doesn't exist, continue + if (!datFile.Items.ContainsKey(key)) + continue; + + List datItems = datFile.Items[key]; + var mappings = machineMap[key]; + + foreach (var datItem in datItems) + { + DatItemTool.SetFields(datItem.Machine, mappings); + } + } + + // Now apply the new set of DatItem mappings + foreach (string key in datItemMap.Keys) + { + // If the key doesn't exist, continue + if (!datFile.Items.ContainsKey(key)) + continue; + + List datItems = datFile.Items[key]; + var mappings = datItemMap[key]; + + foreach (var datItem in datItems) + { + DatItemTool.SetFields(datItem, mappings, null); + } + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + #endregion } } diff --git a/SabreTools.Filtering/MachineFilter.cs b/SabreTools.Filtering/MachineFilter.cs index 99f452fc..c71f27f5 100644 --- a/SabreTools.Filtering/MachineFilter.cs +++ b/SabreTools.Filtering/MachineFilter.cs @@ -112,7 +112,7 @@ namespace SabreTools.Filtering #endregion - #region Filter Population + #region Population /// /// Set multiple filters from key @@ -323,7 +323,7 @@ namespace SabreTools.Filtering #endregion - #region Filter Running + #region Running /// /// Check to see if a Machine passes the filters diff --git a/SabreTools.DatTools/Modification.cs b/SabreTools.Filtering/Splitter.cs similarity index 54% rename from SabreTools.DatTools/Modification.cs rename to SabreTools.Filtering/Splitter.cs index 5c2c2082..c5cecbf6 100644 --- a/SabreTools.DatTools/Modification.cs +++ b/SabreTools.Filtering/Splitter.cs @@ -1,25 +1,25 @@ -using System; -using System.Collections.Concurrent; +using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; using SabreTools.Core; using SabreTools.DatFiles; using SabreTools.DatItems; -using SabreTools.Filtering; -using SabreTools.IO; using SabreTools.Logging; -namespace SabreTools.DatTools +namespace SabreTools.Filtering { - /// - /// Helper methods for cleaning and filtering DatFiles - /// - public class Modification + public class Splitter { + #region Fields + + /// + /// Splitting mode to apply + /// + public MergingFlag SplitType { get; set; } + + #endregion + #region Logging /// @@ -28,548 +28,72 @@ namespace SabreTools.DatTools private static readonly Logger logger = new Logger(); #endregion - - /// - /// Apply cleaning methods to the DatFile - /// - /// Current DatFile object to run operations on - /// Cleaner to use - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if cleaning was successful, false on error - public static bool ApplyCleaning(DatFile datFile, Cleaner cleaner, bool throwOnError = false) - { - try - { - // Perform item-level cleaning - CleanDatItems(datFile, cleaner); - - // Bucket and dedupe according to the flag - if (cleaner?.DedupeRoms == DedupeType.Full) - datFile.Items.BucketBy(ItemKey.CRC, cleaner.DedupeRoms); - else if (cleaner?.DedupeRoms == DedupeType.Game) - datFile.Items.BucketBy(ItemKey.Machine, cleaner.DedupeRoms); - - // Process description to machine name - if (cleaner?.DescriptionAsName == true) - MachineDescriptionToName(datFile); - - // If we are removing scene dates, do that now - if (cleaner?.SceneDateStrip == true) - StripSceneDatesFromItems(datFile); - - // Run the one rom per game logic, if required - if (cleaner?.OneGamePerRegion == true) - OneGamePerRegion(datFile, cleaner.RegionList); - - // Run the one rom per game logic, if required - if (cleaner?.OneRomPerGame == true) - OneRomPerGame(datFile); - - // If we are removing fields, do that now - cleaner.RemoveFieldsFromItems(datFile); - - // Remove all marked items - datFile.Items.ClearMarked(); - - // We remove any blanks, if we aren't supposed to have any - if (cleaner?.KeepEmptyGames == false) - datFile.Items.ClearEmpty(); - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Apply a set of Extra INIs on the DatFile - /// - /// Current DatFile object to run operations on - /// ExtrasIni to use - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the extras were applied, false on error - public static bool ApplyExtras(DatFile datFile, ExtraIni extras, bool throwOnError = false) - { - try - { - // Bucket by game first - datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); - - // Create a new set of mappings based on the items - var machineMap = new Dictionary>(); - var datItemMap = new Dictionary>(); - - // Loop through each of the extras - foreach (ExtraIniItem item in extras.Items) - { - foreach (var mapping in item.Mappings) - { - string key = mapping.Key; - List machineNames = mapping.Value; - - // Loop through the machines and add the new mappings - foreach (string machine in machineNames) - { - if (item.MachineField != MachineField.NULL) - { - if (!machineMap.ContainsKey(machine)) - machineMap[machine] = new Dictionary(); - - machineMap[machine][item.MachineField] = key; - } - else if (item.DatItemField != DatItemField.NULL) - { - if (!datItemMap.ContainsKey(machine)) - datItemMap[machine] = new Dictionary(); - - datItemMap[machine][item.DatItemField] = key; - } - } - } - } - - // Now apply the new set of Machine mappings - foreach (string key in machineMap.Keys) - { - // If the key doesn't exist, continue - if (!datFile.Items.ContainsKey(key)) - continue; - - List datItems = datFile.Items[key]; - var mappings = machineMap[key]; - - foreach (var datItem in datItems) - { - DatItemTool.SetFields(datItem.Machine, mappings); - } - } - - // Now apply the new set of DatItem mappings - foreach (string key in datItemMap.Keys) - { - // If the key doesn't exist, continue - if (!datFile.Items.ContainsKey(key)) - continue; - - List datItems = datFile.Items[key]; - var mappings = datItemMap[key]; - - foreach (var datItem in datItems) - { - DatItemTool.SetFields(datItem, mappings, null); - } - } - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Apply a set of Filters on the DatFile - /// - /// Current DatFile object to run operations on - /// Cleaner to use - /// True if entire machines are considered, false otherwise (default) - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the DatFile was filtered, false on error - public static bool ApplyFilters(DatFile datFile, Cleaner cleaner, bool perMachine = false, bool throwOnError = false) - { - // If we have a null cleaner or filters, return false - if (cleaner == null || cleaner.MachineFilter == null || cleaner.DatItemFilter == null) - return false; - - // If we're filtering per machine, bucket by machine first - if (perMachine) - datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); - - try - { - // Loop over every key in the dictionary - List keys = datFile.Items.Keys.ToList(); - foreach (string key in keys) - { - // For every item in the current key - bool machinePass = true; - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If we have a null item, we can't pass it - if (item == null) - continue; - - // If the item is already filtered out, we skip - if (item.Remove) - continue; - - // If the rom doesn't pass the filter, mark for removal - if (!cleaner.PassesFilters(item)) - { - item.Remove = true; - - // If we're in machine mode, set and break - if (perMachine) - { - machinePass = false; - break; - } - } - } - - // If we didn't pass and we're in machine mode, set all items as remove - if (perMachine && !machinePass) - { - foreach (DatItem item in items) - { - item.Remove = true; - } - } - - // Assign back for caution - datFile.Items[key] = items; - } - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Apply splitting on the DatFile - /// - /// Current DatFile object to run operations on - /// Split type to try - /// True if DatFile tags override splitting, false otherwise - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the DatFile was split, false on error - public static bool ApplySplitting(DatFile datFile, MergingFlag splitType, bool useTags, bool throwOnError = false) - { - try - { - // If we are using tags from the DAT, set the proper input for split type unless overridden - if (useTags && splitType == MergingFlag.None) - splitType = datFile.Header.ForceMerging; - - // Run internal splitting - switch (splitType) - { - case MergingFlag.None: - // No-op - break; - case MergingFlag.Device: - CreateDeviceNonMergedSets(datFile, DedupeType.None); - break; - case MergingFlag.Full: - CreateFullyNonMergedSets(datFile, DedupeType.None); - break; - case MergingFlag.NonMerged: - CreateNonMergedSets(datFile, DedupeType.None); - break; - case MergingFlag.Merged: - CreateMergedSets(datFile, DedupeType.None); - break; - case MergingFlag.Split: - CreateSplitSets(datFile, DedupeType.None); - break; - } - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Apply SuperDAT naming logic to a merged DatFile - /// - /// Current DatFile object to run operations on - /// List of inputs to use for renaming - public static void ApplySuperDAT(DatFile datFile, List inputs) - { - List keys = datFile.Items.Keys.ToList(); - Parallel.ForEach(keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key].ToList(); - List newItems = new List(); - foreach (DatItem item in items) - { - DatItem newItem = item; - string filename = inputs[newItem.Source.Index].CurrentPath; - string rootpath = inputs[newItem.Source.Index].ParentPath; - - if (!string.IsNullOrWhiteSpace(rootpath)) - rootpath += Path.DirectorySeparatorChar.ToString(); - - filename = filename.Remove(0, rootpath.Length); - newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar - + Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar - + newItem.Machine.Name; - - newItems.Add(newItem); - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newItems); - }); - } - - /// - /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof - /// - /// Current DatFile object to run operations on - /// True if the error that is thrown should be thrown back to the caller, false otherwise - public static void MachineDescriptionToName(DatFile datFile, bool throwOnError = false) - { - try - { - // First we want to get a mapping for all games to description - ConcurrentDictionary mapping = new ConcurrentDictionary(); - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If the key mapping doesn't exist, add it - mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); - } - }); - - // Now we loop through every item and update accordingly - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - List newItems = new List(); - foreach (DatItem item in items) - { - // Update machine name - if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name)) - item.Machine.Name = mapping[item.Machine.Name]; - - // Update cloneof - if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf)) - item.Machine.CloneOf = mapping[item.Machine.CloneOf]; - - // Update romof - if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf)) - item.Machine.RomOf = mapping[item.Machine.RomOf]; - - // Update sampleof - if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf)) - item.Machine.SampleOf = mapping[item.Machine.SampleOf]; - - // Add the new item to the output list - newItems.Add(item); - } - - // Replace the old list of roms with the new one - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newItems); - }); - } - catch (Exception ex) when (!throwOnError) - { - logger.Warning(ex.ToString()); - } - } - - /// - /// Filter a DAT using 1G1R logic given an ordered set of regions - /// - /// Current DatFile object to run operations on - /// Ordered list of regions to use - /// - /// In the most technical sense, the way that the region list is being used does not - /// confine its values to be just regions. Since it's essentially acting like a - /// specialized version of the machine name filter, anything that is usually encapsulated - /// in parenthesis would be matched on, including disc numbers, languages, editions, - /// and anything else commonly used. Please note that, unlike other existing 1G1R - /// solutions, this does not have the ability to contain custom mappings of parent - /// to clone sets based on name, nor does it have the ability to match on the - /// Release DatItem type. - /// - public static void OneGamePerRegion(DatFile datFile, List regions) - { - // If we have null region list, make it empty - if (regions == null) - regions = new List(); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); - - // Then we want to get a mapping of all machines to parents - Dictionary> parents = new Dictionary>(); - foreach (string key in datFile.Items.Keys) - { - DatItem item = datFile.Items[key][0]; - - // Match on CloneOf first - if (!string.IsNullOrEmpty(item.Machine.CloneOf)) - { - if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant())) - parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List()); - - parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); - } - - // Then by RomOf - else if (!string.IsNullOrEmpty(item.Machine.RomOf)) - { - if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant())) - parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List()); - - parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); - } - - // Otherwise, treat it as a parent - else - { - if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant())) - parents.Add(item.Machine.Name.ToLowerInvariant(), new List()); - - parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); - } - } - - // Once we have the full list of mappings, filter out games to keep - foreach (string key in parents.Keys) - { - // Find the first machine that matches the regions in order, if possible - string machine = default; - foreach (string region in regions) - { - machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase)); - if (machine != default) - break; - } - - // If we didn't get a match, use the parent - if (machine == default) - machine = key; - - // Remove the key from the list - parents[key].Remove(machine); - - // Remove the rest of the items from this key - parents[key].ForEach(k => datFile.Items.Remove(k)); - } - - // Finally, strip out the parent tags - RemoveTagsFromChild(datFile); - } - - /// - /// Ensure that all roms are in their own game (or at least try to ensure) - /// - /// Current DatFile object to run operations on - public static void OneRomPerGame(DatFile datFile) - { - // Because this introduces subfolders, we need to set the SuperDAT type - datFile.Header.Type = "SuperDAT"; - - // For each rom, we want to update the game to be "/" - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int i = 0; i < items.Count; i++) - { - DatItemTool.SetOneRomPerGame(items[i]); - } - }); - } - - /// - /// Strip the dates from the beginning of scene-style set names - /// - /// Current DatFile object to run operations on - public static void StripSceneDatesFromItems(DatFile datFile) - { - // Output the logging statement - logger.User("Stripping scene-style dates"); - - // Set the regex pattern to use - string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)"; - - // Now process all of the roms - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int j = 0; j < items.Count; j++) - { - DatItem item = items[j]; - if (Regex.IsMatch(item.Machine.Name, pattern)) - item.Machine.Name = Regex.Replace(item.Machine.Name, pattern, "$2"); - - if (Regex.IsMatch(item.Machine.Description, pattern)) - item.Machine.Description = Regex.Replace(item.Machine.Description, pattern, "$2"); - - items[j] = item; - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, items); - }); - } - /// - /// Clean individual items based on the current filter - /// - /// Current DatFile object to run operations on - /// Cleaner to use - private static void CleanDatItems(DatFile datFile, Cleaner cleaner) - { - List keys = datFile.Items.Keys.ToList(); - foreach (string key in keys) - { - // For every item in the current key - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If we have a null item, we can't clean it it - if (item == null) - continue; - - // Run cleaning per item - cleaner.CleanDatItem(item); - } - - // Assign back for caution - datFile.Items[key] = items; - } - } - // TODO: Should any of these create a new DatFile in the process? // The reason this comes up is that doing any of the splits or merges // is an inherently destructive process. Making it output a new DatFile // might make it easier to deal with multiple internal steps. On the other // hand, this will increase memory usage significantly and would force the // existing paths to behave entirely differently - #region Internal Splitting/Merging + #region Running + + /// + /// Apply splitting on the DatFile + /// + /// Current DatFile object to run operations on + /// True if DatFile tags override splitting, false otherwise + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if the DatFile was split, false on error + public bool ApplySplitting(DatFile datFile, bool useTags, bool throwOnError = false) + { + try + { + // If we are using tags from the DAT, set the proper input for split type unless overridden + if (useTags && SplitType == MergingFlag.None) + SplitType = datFile.Header.ForceMerging; + + // Run internal splitting + switch (SplitType) + { + case MergingFlag.None: + // No-op + break; + case MergingFlag.Device: + CreateDeviceNonMergedSets(datFile); + break; + case MergingFlag.Full: + CreateFullyNonMergedSets(datFile); + break; + case MergingFlag.NonMerged: + CreateNonMergedSets(datFile); + break; + case MergingFlag.Merged: + CreateMergedSets(datFile); + break; + case MergingFlag.Split: + CreateSplitSets(datFile); + break; + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } /// /// Use cdevice_ref tags to get full non-merged sets and remove parenting tags /// /// Current DatFile object to run operations on - /// Dedupe type to be used - private static void CreateDeviceNonMergedSets(DatFile datFile, DedupeType mergeroms) + internal static void CreateDeviceNonMergedSets(DatFile datFile) { logger.User("Creating device non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true); + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information while (AddRomsFromDevices(datFile, false, false)) ; @@ -583,13 +107,12 @@ namespace SabreTools.DatTools /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets /// /// Current DatFile object to run operations on - /// Dedupe type to be used - private static void CreateFullyNonMergedSets(DatFile datFile, DedupeType mergeroms) + internal static void CreateFullyNonMergedSets(DatFile datFile) { logger.User("Creating fully non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true); + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information while (AddRomsFromDevices(datFile, true, true)) ; @@ -607,13 +130,12 @@ namespace SabreTools.DatTools /// Use cloneof tags to create merged sets and remove the tags /// /// Current DatFile object to run operations on - /// Dedupe type to be used - private static void CreateMergedSets(DatFile datFile, DedupeType mergeroms) + internal static void CreateMergedSets(DatFile datFile) { logger.User("Creating merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true); + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information AddRomsFromChildren(datFile); @@ -630,13 +152,12 @@ namespace SabreTools.DatTools /// Use cloneof tags to create non-merged sets and remove the tags /// /// Current DatFile object to run operations on - /// Dedupe type to be used - private static void CreateNonMergedSets(DatFile datFile, DedupeType mergeroms) + internal static void CreateNonMergedSets(DatFile datFile) { logger.User("Creating non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true); + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information AddRomsFromParent(datFile); @@ -653,13 +174,12 @@ namespace SabreTools.DatTools /// Use cloneof and romof tags to create split sets and remove the tags /// /// Current DatFile object to run operations on - /// Dedupe type to be used - private static void CreateSplitSets(DatFile datFile, DedupeType mergeroms) + internal static void CreateSplitSets(DatFile datFile) { logger.User("Creating split sets from the DAT"); // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(ItemKey.Machine, mergeroms, norename: true); + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information RemoveRomsFromChild(datFile); @@ -676,7 +196,7 @@ namespace SabreTools.DatTools /// Use romof tags to add roms to the children /// /// Current DatFile object to run operations on - private static void AddRomsFromBios(DatFile datFile) + internal static void AddRomsFromBios(DatFile datFile) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -717,7 +237,7 @@ namespace SabreTools.DatTools /// Current DatFile object to run operations on /// True if only child device sets are touched, false for non-device sets (default) /// True if slotoptions tags are used as well, false otherwise - private static bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool useSlotOptions = false) + internal static bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool useSlotOptions = false) { bool foundnew = false; List machines = datFile.Items.Keys.OrderBy(g => g).ToList(); @@ -845,7 +365,7 @@ namespace SabreTools.DatTools /// Use cloneof tags to add roms to the children, setting the new romof tag in the process /// /// Current DatFile object to run operations on - private static void AddRomsFromParent(DatFile datFile) + internal static void AddRomsFromParent(DatFile datFile) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -896,7 +416,7 @@ namespace SabreTools.DatTools /// /// Current DatFile object to run operations on /// True to add DatItems to subfolder of parent (not including Disk), false otherwise - private static void AddRomsFromChildren(DatFile datFile, bool subfolder = true) + internal static void AddRomsFromChildren(DatFile datFile, bool subfolder = true) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -1008,7 +528,7 @@ namespace SabreTools.DatTools /// Remove all BIOS and device sets /// /// Current DatFile object to run operations on - private static void RemoveBiosAndDeviceSets(DatFile datFile) + internal static void RemoveBiosAndDeviceSets(DatFile datFile) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -1027,7 +547,7 @@ namespace SabreTools.DatTools /// /// Current DatFile object to run operations on /// True if only child Bios sets are touched, false for non-bios sets (default) - private static void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false) + internal static void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false) { // Loop through the romof tags List games = datFile.Items.Keys.OrderBy(g => g).ToList(); @@ -1071,7 +591,7 @@ namespace SabreTools.DatTools /// Use cloneof tags to remove roms from the children /// /// Current DatFile object to run operations on - private static void RemoveRomsFromChild(DatFile datFile) + internal static void RemoveRomsFromChild(DatFile datFile) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -1118,7 +638,7 @@ namespace SabreTools.DatTools /// Remove all romof and cloneof tags from all games /// /// Current DatFile object to run operations on - private static void RemoveTagsFromChild(DatFile datFile) + internal static void RemoveTagsFromChild(DatFile datFile) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -1135,4 +655,4 @@ namespace SabreTools.DatTools #endregion } -} \ No newline at end of file +} diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index 5f682b20..f06b8a3b 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -1735,6 +1735,11 @@ Some special strings that can be used: /// protected string OutputDir { get; set; } + /// + /// Pre-configured Splitter + /// + protected Filtering.Splitter Splitter { get; set; } + #endregion #region Add Feature Groups @@ -1801,6 +1806,7 @@ Some special strings that can be used: Extras = GetExtras(features); Header = GetDatHeader(features); OutputDir = GetString(features, OutputDirStringValue); + Splitter = GetSplitter(features); // Set threading flag, if necessary if (features.ContainsKey(ThreadsInt32Value)) @@ -1895,26 +1901,6 @@ Some special strings that can be used: return splittingMode; } - /// - /// Get SplitType from feature list - /// - protected MergingFlag GetSplitType(Dictionary features) - { - MergingFlag splitType = MergingFlag.None; - if (GetBoolean(features, DatDeviceNonMergedValue)) - splitType = MergingFlag.Device; - else if (GetBoolean(features, DatFullNonMergedValue)) - splitType = MergingFlag.Full; - else if (GetBoolean(features, DatMergedValue)) - splitType = MergingFlag.Merged; - else if (GetBoolean(features, DatNonMergedValue)) - splitType = MergingFlag.NonMerged; - else if (GetBoolean(features, DatSplitValue)) - splitType = MergingFlag.Split; - - return splitType; - } - /// /// Get StatReportFormat from feature list /// @@ -2135,6 +2121,38 @@ Some special strings that can be used: return extraIni; } + /// + /// Get Splitter from feature list + /// + private Filtering.Splitter GetSplitter(Dictionary features) + { + Filtering.Splitter splitter = new Filtering.Splitter + { + SplitType = GetSplitType(features), + }; + return splitter; + } + + /// + /// Get SplitType from feature list + /// + private MergingFlag GetSplitType(Dictionary features) + { + MergingFlag splitType = MergingFlag.None; + if (GetBoolean(features, DatDeviceNonMergedValue)) + splitType = MergingFlag.Device; + else if (GetBoolean(features, DatFullNonMergedValue)) + splitType = MergingFlag.Full; + else if (GetBoolean(features, DatMergedValue)) + splitType = MergingFlag.Merged; + else if (GetBoolean(features, DatNonMergedValue)) + splitType = MergingFlag.NonMerged; + else if (GetBoolean(features, DatSplitValue)) + splitType = MergingFlag.Split; + + return splitType; + } + #endregion #region Protected Helpers diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index ebc14cbd..4be2eed3 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -168,7 +168,7 @@ Reset the internal state: reset();"; "DatItem.SpamSum", "DatItem.Date", }); - Modification.ApplyCleaning(datFile, dfdCleaner); + dfdCleaner.ApplyCleaning(datFile); break; @@ -213,18 +213,18 @@ Reset the internal state: reset();"; } // Create cleaner to run filters from - Cleaner cleaner = new Cleaner + Cleaner filterCleaner = new Cleaner { MachineFilter = new MachineFilter(), DatItemFilter = new DatItemFilter(), }; // Set the possible filters - cleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); - cleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); + filterCleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); + filterCleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); // Apply the filters blindly - Modification.ApplyFilters(datFile, cleaner, filterPerMachine.Value); + filterCleaner.ApplyFilters(datFile, filterPerMachine.Value); // Cleanup after the filter // TODO: We might not want to remove immediately @@ -269,7 +269,7 @@ Reset the internal state: reset();"; extraIni.Items.Add(extraIniItem); // Apply the extra INI blindly - Modification.ApplyExtras(datFile, extraIni); + extraIni.ApplyExtras(datFile); break; @@ -293,7 +293,8 @@ Reset the internal state: reset();"; } // Apply the merging flag - Modification.ApplySplitting(datFile, mergingFlag, false); + Filtering.Splitter splitter = new Filtering.Splitter { SplitType = mergingFlag }; + splitter.ApplySplitting(datFile, false); break; @@ -307,7 +308,8 @@ Reset the internal state: reset();"; } // Apply the logic - Modification.MachineDescriptionToName(datFile); + Cleaner descNameCleaner = new Cleaner { DescriptionAsName = true }; + descNameCleaner.ApplyCleaning(datFile); break; @@ -321,7 +323,8 @@ Reset the internal state: reset();"; } // Run the 1G1R functionality - Modification.OneGamePerRegion(datFile, command.Arguments); + Cleaner ogorCleaner = new Cleaner { OneGamePerRegion = true, RegionList = command.Arguments }; + ogorCleaner.ApplyCleaning(datFile); break; @@ -335,7 +338,8 @@ Reset the internal state: reset();"; } // Apply the logic - Modification.OneRomPerGame(datFile); + Cleaner orpgCleaner = new Cleaner { OneRomPerGame = true }; + orpgCleaner.ApplyCleaning(datFile); break; @@ -365,7 +369,8 @@ Reset the internal state: reset();"; } // Apply the logic - Modification.StripSceneDatesFromItems(datFile); + Cleaner stripCleaner = new Cleaner { SceneDateStrip = true }; + stripCleaner.ApplyCleaning(datFile); break; diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index ab6b7a6b..538788e6 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; -using SabreTools.Core; using SabreTools.DatFiles; using SabreTools.DatTools; using SabreTools.FileTypes; @@ -62,10 +61,10 @@ namespace SabreTools.Features bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue); var includeInScan = GetIncludeInScan(features); var skipFileType = GetSkipFileType(features); - var splitType = GetSplitType(features); // Apply the specialized field removals to the cleaner - Cleaner.PopulateExclusionsFromList(new List { "DatItem.Date" }); + if (!addFileDates) + Cleaner.PopulateExclusionsFromList(new List { "DatItem.Date" }); // Create a new DATFromDir object and process the inputs DatFile basedat = DatFile.Create(Header); @@ -95,10 +94,10 @@ namespace SabreTools.Features if (success) { // Perform additional processing steps - Modification.ApplyExtras(datdata, Extras); - Modification.ApplySplitting(datdata, splitType, false); - Modification.ApplyFilters(datdata, Cleaner); - Modification.ApplyCleaning(datdata, Cleaner); + Extras.ApplyExtras(datdata); + Splitter.ApplySplitting(datdata, false); + Cleaner.ApplyFilters(datdata); + Cleaner.ApplyCleaning(datdata); // Write out the file Writer.Write(datdata, OutputDir); diff --git a/SabreTools/Features/Split.cs b/SabreTools/Features/Split.cs index 1d67fc29..6be4aefe 100644 --- a/SabreTools/Features/Split.cs +++ b/SabreTools/Features/Split.cs @@ -65,7 +65,7 @@ namespace SabreTools.Features // Extension splitting if (splittingMode.HasFlag(SplittingMode.Extension)) { - (DatFile extADat, DatFile extBDat) = Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue)); + (DatFile extADat, DatFile extBDat) = DatTools.Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue)); InternalStopwatch watch = new InternalStopwatch("Outputting extension-split DATs"); @@ -79,7 +79,7 @@ namespace SabreTools.Features // Hash splitting if (splittingMode.HasFlag(SplittingMode.Hash)) { - Dictionary typeDats = Splitter.SplitByHash(internalDat); + Dictionary typeDats = DatTools.Splitter.SplitByHash(internalDat); InternalStopwatch watch = new InternalStopwatch("Outputting hash-split DATs"); @@ -96,7 +96,7 @@ namespace SabreTools.Features if (splittingMode.HasFlag(SplittingMode.Level)) { logger.Warning("This feature is not implemented: level-split"); - Splitter.SplitByLevel( + DatTools.Splitter.SplitByLevel( internalDat, OutputDir, GetBoolean(features, ShortValue), @@ -106,7 +106,7 @@ namespace SabreTools.Features // Size splitting if (splittingMode.HasFlag(SplittingMode.Size)) { - (DatFile lessThan, DatFile greaterThan) = Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value)); + (DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value)); InternalStopwatch watch = new InternalStopwatch("Outputting size-split DATs"); @@ -120,7 +120,7 @@ namespace SabreTools.Features // Type splitting if (splittingMode.HasFlag(SplittingMode.Type)) { - Dictionary typeDats = Splitter.SplitByType(internalDat); + Dictionary typeDats = DatTools.Splitter.SplitByType(internalDat); InternalStopwatch watch = new InternalStopwatch("Outputting ItemType DATs"); diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index bcd201e5..88df3ae7 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -156,10 +156,10 @@ namespace SabreTools.Features || datFile.Header.DatFormat.HasFlag(DatFormat.SSV)); // Perform additional processing steps - Modification.ApplyExtras(datFile, Extras); - Modification.ApplySplitting(datFile, GetSplitType(features), false); - Modification.ApplyFilters(datFile, Cleaner); - Modification.ApplyCleaning(datFile, Cleaner); + Extras.ApplyExtras(datFile); + Splitter.ApplySplitting(datFile, false); + Cleaner.ApplyFilters(datFile); + Cleaner.ApplyCleaning(datFile); // Get the correct output path string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue)); @@ -194,10 +194,10 @@ namespace SabreTools.Features datHeaders = DatFileTool.PopulateUserData(userInputDat, inputPaths); // Perform additional processing steps - Modification.ApplyExtras(userInputDat, Extras); - Modification.ApplySplitting(userInputDat, GetSplitType(features), false); - Modification.ApplyFilters(userInputDat, Cleaner); - Modification.ApplyCleaning(userInputDat, Cleaner); + Extras.ApplyExtras(userInputDat); + Splitter.ApplySplitting(userInputDat, false); + Cleaner.ApplyFilters(userInputDat); + Cleaner.ApplyCleaning(userInputDat); // Output only DatItems that are duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) @@ -286,10 +286,10 @@ namespace SabreTools.Features Parser.ParseInto(repDat, inputPath, indexId: 1, keep: true); // Perform additional processing steps - Modification.ApplyExtras(repDat, Extras); - Modification.ApplySplitting(repDat, GetSplitType(features), false); - Modification.ApplyFilters(repDat, Cleaner); - Modification.ApplyCleaning(repDat, Cleaner); + Extras.ApplyExtras(repDat); + Splitter.ApplySplitting(repDat, false); + Cleaner.ApplyFilters(repDat); + Cleaner.ApplyCleaning(repDat); // Now replace the fields from the base DatFile DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue)); @@ -311,10 +311,10 @@ namespace SabreTools.Features Parser.ParseInto(repDat, inputPath, indexId: 1, keep: true); // Perform additional processing steps - Modification.ApplyExtras(repDat, Extras); - Modification.ApplySplitting(repDat, GetSplitType(features), false); - Modification.ApplyFilters(repDat, Cleaner); - Modification.ApplyCleaning(repDat, Cleaner); + Extras.ApplyExtras(repDat); + Splitter.ApplySplitting(repDat, false); + Cleaner.ApplyFilters(repDat); + Cleaner.ApplyCleaning(repDat); // Now replace the fields from the base DatFile DatFileTool.BaseReplace( @@ -336,7 +336,7 @@ namespace SabreTools.Features { // If we're in SuperDAT mode, prefix all games with their respective DATs if (string.Equals(userInputDat.Header.Type, "SuperDAT", StringComparison.OrdinalIgnoreCase)) - Modification.ApplySuperDAT(userInputDat, inputPaths); + DatFileTool.ApplySuperDAT(userInputDat, inputPaths); Writer.Write(userInputDat, OutputDir); } diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index ea2ba55c..67390be2 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -50,7 +50,6 @@ namespace SabreTools.Features TreatAsFile asFiles = GetTreatAsFiles(features); bool hashOnly = GetBoolean(features, HashOnlyValue); bool quickScan = GetBoolean(features, QuickValue); - var splitType = GetSplitType(features); // If we are in individual mode, process each DAT on their own if (GetBoolean(features, IndividualValue)) @@ -62,10 +61,10 @@ namespace SabreTools.Features Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true); // Perform additional processing steps - Modification.ApplyExtras(datdata, Extras); - Modification.ApplySplitting(datdata, splitType, true); - Modification.ApplyFilters(datdata, Cleaner); - Modification.ApplyCleaning(datdata, Cleaner); + Extras.ApplyExtras(datdata); + Splitter.ApplySplitting(datdata, true); + Cleaner.ApplyFilters(datdata); + Cleaner.ApplyCleaning(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; @@ -109,10 +108,10 @@ namespace SabreTools.Features } // Perform additional processing steps - Modification.ApplyExtras(datdata, Extras); - Modification.ApplySplitting(datdata, splitType, true); - Modification.ApplyFilters(datdata, Cleaner); - Modification.ApplyCleaning(datdata, Cleaner); + Extras.ApplyExtras(datdata); + Splitter.ApplySplitting(datdata, true); + Cleaner.ApplyFilters(datdata); + Cleaner.ApplyCleaning(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;