From a82b7ccab8fe06f4ac07f4bb695bd8090e6d20c5 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Mon, 1 Feb 2021 12:11:32 -0800 Subject: [PATCH] Remover code to own class This was the last of the major bits of code that was technically distinct from Cleaning. This means that each of the bits of functionality that used to all be in Cleaner are split out into their approrpriate classes. --- RombaSharp/Features/Dir2Dat.cs | 6 +- SabreTools.Filtering/Cleaner.cs | 275 +------------------ SabreTools.Filtering/DatHeaderRemover.cs | 21 +- SabreTools.Filtering/DatItemRemover.cs | 21 +- SabreTools.Filtering/Filter.cs | 160 ++++++++++- SabreTools.Filtering/Remover.cs | 95 ++++++- SabreTools.Test/Filtering/FilteringTests.cs | 32 +-- SabreTools.Test/Filtering/PopulationTests.cs | 104 +++---- SabreTools/Features/BaseFeature.cs | 62 ++++- SabreTools/Features/Batch.cs | 20 +- SabreTools/Features/DatFromDir.cs | 5 +- SabreTools/Features/Update.cs | 12 +- SabreTools/Features/Verify.cs | 6 +- 13 files changed, 405 insertions(+), 414 deletions(-) diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index 7836a4b1..a7faa733 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -56,15 +56,15 @@ namespace RombaSharp.Features datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name; datfile.Header.Description = description; DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive); - Cleaner cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(new List + Remover remover = new Remover(); + remover.PopulateExclusionsFromList(new List { "DatItem.SHA256", "DatItem.SHA384", "DatItem.SHA512", "DatItem.SpamSum", }); - cleaner.ApplyCleaning(datfile); + remover.ApplyRemovals(datfile); Writer.Write(datfile, outdat); } } diff --git a/SabreTools.Filtering/Cleaner.cs b/SabreTools.Filtering/Cleaner.cs index b1635780..f182ffca 100644 --- a/SabreTools.Filtering/Cleaner.cs +++ b/SabreTools.Filtering/Cleaner.cs @@ -8,7 +8,6 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using SabreTools.Core; -using SabreTools.Core.Tools; using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.Logging; @@ -22,35 +21,7 @@ namespace SabreTools.Filtering public class Cleaner { - #region Exclusion Fields - - /// - /// DatItemRemover to remove fields from DatHeaders - /// - public DatHeaderRemover DatHeaderRemover { get; set; } - - /// - /// DatItemRemover to remove fields from DatItems - /// - public DatItemRemover DatItemRemover { get; set; } - - #endregion - - #region Filter Fields - - /// - /// Filter for DatItem fields - /// - public DatItemFilter DatItemFilter { get; set; } - - /// - /// Filter for Machine fields - /// - public MachineFilter MachineFilter { get; set; } - - #endregion - - #region Flag Fields + #region Fields /// /// Clean all names to WoD standards @@ -123,87 +94,7 @@ namespace SabreTools.Filtering #endregion - #region Population - - /// - /// Populate the exclusion objects using a set of field names - /// - /// List of field names - public void PopulateExclusionsFromList(List fields) - { - // Instantiate the removers, if necessary - DatHeaderRemover ??= new DatHeaderRemover(); - DatItemRemover ??= new DatItemRemover(); - - // If the list is null or empty, just return - if (fields == null || fields.Count == 0) - return; - - foreach (string field in fields) - { - // If we don't even have a possible field name - if (field == null) - continue; - - // DatHeader fields - if (DatHeaderRemover.SetRemover(field)) - continue; - - // Machine and DatItem fields - if (DatItemRemover.SetRemover(field)) - continue; - - // If we didn't match anything, log an error - logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names."); - } - } - - /// - /// Populate the filters objects using a set of key:value filters - /// - /// List of key:value where ~key/!key is negated - public void PopulateFiltersFromList(List filters) - { - // Instantiate the filters, if necessary - MachineFilter ??= new MachineFilter(); - DatItemFilter ??= new DatItemFilter(); - - // If the list is null or empty, just return - if (filters == null || filters.Count == 0) - return; - - foreach (string filterPair in filters) - { - (string field, string value, bool negate) = ProcessFilterPair(filterPair); - - // If we don't even have a possible filter pair - if (field == null && value == null) - continue; - - // Machine fields - MachineField machineField = field.AsMachineField(); - if (machineField != MachineField.NULL) - { - MachineFilter.SetFilter(machineField, value, negate); - continue; - } - - // DatItem fields - DatItemField datItemField = field.AsDatItemField(); - if (datItemField != DatItemField.NULL) - { - DatItemFilter.SetFilter(datItemField, value, negate); - continue; - } - - // If we didn't match anything, log an error - logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names."); - } - } - - #endregion - - #region Cleaning + #region Running /// /// Apply cleaning methods to the DatFile @@ -240,9 +131,6 @@ namespace SabreTools.Filtering if (OneRomPerGame == true) SetOneRomPerGame(datFile); - // If we are removing fields, do that now - RemoveFieldsFromItems(datFile); - // Remove all marked items datFile.Items.ClearMarked(); @@ -691,164 +579,5 @@ namespace SabreTools.Filtering } #endregion - - #region Filtering - - /// - /// Apply a set of Filters on the DatFile - /// - /// Current DatFile object to run operations on - /// True if entire machines are considered, false otherwise (default) - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the DatFile was filtered, false on error - public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false) - { - // If we have null filters, return false - if (MachineFilter == null || DatItemFilter == null) - return false; - - // If we're filtering per machine, bucket by machine first - if (perMachine) - datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); - - try - { - // Loop over every key in the dictionary - List keys = datFile.Items.Keys.ToList(); - foreach (string key in keys) - { - // For every item in the current key - bool machinePass = true; - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If we have a null item, we can't pass it - if (item == null) - continue; - - // If the item is already filtered out, we skip - if (item.Remove) - continue; - - // If the rom doesn't pass the filter, mark for removal - if (!PassesFilters(item)) - { - item.Remove = true; - - // If we're in machine mode, set and break - if (perMachine) - { - machinePass = false; - break; - } - } - } - - // If we didn't pass and we're in machine mode, set all items as remove - if (perMachine && !machinePass) - { - foreach (DatItem item in items) - { - item.Remove = true; - } - } - - // Assign back for caution - datFile.Items[key] = items; - } - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Check to see if a DatItem passes the filters - /// - /// DatItem to check - /// True if the item passed the filter, false otherwise - internal bool PassesFilters(DatItem datItem) - { - // Null item means it will never pass - if (datItem == null) - return false; - - // Filter on Machine fields - if (!MachineFilter.PassesFilters(datItem.Machine)) - return false; - - // Filter on DatItem fields - return DatItemFilter.PassesFilters(datItem); - } - - /// - /// Split the parts of a filter statement - /// - /// key:value where ~key/!key is negated - private (string field, string value, bool negate) ProcessFilterPair(string filter) - { - // If we don't even have a possible filter pair - if (!filter.Contains(":")) - { - logger.Warning($"'{filter}` is not a valid filter string. Valid filter strings are of the form 'key:value'. Please refer to README.1ST or the help feature for more details."); - return (null, null, false); - } - - string filterTrimmed = filter.Trim('"', ' ', '\t'); - bool negate = filterTrimmed.StartsWith("!") - || filterTrimmed.StartsWith("~") - || filterTrimmed.StartsWith("not-"); - filterTrimmed = filterTrimmed.TrimStart('!', '~'); - filterTrimmed = filterTrimmed.StartsWith("not-") ? filterTrimmed[4..] : filterTrimmed; - - string filterFieldString = filterTrimmed.Split(':')[0].ToLowerInvariant().Trim('"', ' ', '\t'); - string filterValue = filterTrimmed[(filterFieldString.Length + 1)..].Trim('"', ' ', '\t'); - - return (filterFieldString, filterValue, negate); - } - - #endregion - - #region Removal - - /// - /// Remove fields as per the header - /// - /// Current DatFile object to run operations on - public void RemoveFieldsFromItems(DatFile datFile) - { - // If the removers don't exist, we can't use it - if (DatHeaderRemover == null && DatItemRemover == null) - return; - - // Output the logging statement - logger.User("Removing filtered fields"); - - // Remove DatHeader fields - if (DatHeaderRemover != null) - DatHeaderRemover.RemoveFields(datFile.Header); - - // Remove DatItem and Machine fields - if (DatItemRemover != null) - { - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int j = 0; j < items.Count; j++) - { - DatItemRemover.RemoveFields(items[j]); - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, items); - }); - } - } - - #endregion } } diff --git a/SabreTools.Filtering/DatHeaderRemover.cs b/SabreTools.Filtering/DatHeaderRemover.cs index 18bf1a02..1d10e0da 100644 --- a/SabreTools.Filtering/DatHeaderRemover.cs +++ b/SabreTools.Filtering/DatHeaderRemover.cs @@ -10,7 +10,7 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a DatHeader /// - public class DatHeaderRemover : Remover + public class DatHeaderRemover { #region Fields @@ -21,22 +21,13 @@ namespace SabreTools.Filtering #endregion - #region Constructors - - /// - /// Constructor - /// - public DatHeaderRemover() - { - logger = new Logger(this); - } - - #endregion - #region Population - /// - public override bool SetRemover(string field) + /// + /// Set remover from a value + /// + /// Key for the remover to be set + public bool SetRemover(string field) { // If the key is null or empty, return false if (string.IsNullOrWhiteSpace(field)) diff --git a/SabreTools.Filtering/DatItemRemover.cs b/SabreTools.Filtering/DatItemRemover.cs index c6341093..17b2e80a 100644 --- a/SabreTools.Filtering/DatItemRemover.cs +++ b/SabreTools.Filtering/DatItemRemover.cs @@ -10,7 +10,7 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a set of items, usually a DAT /// - public class DatItemRemover : Remover + public class DatItemRemover { #region Fields @@ -26,22 +26,13 @@ namespace SabreTools.Filtering #endregion - #region Constructors - - /// - /// Constructor - /// - public DatItemRemover() - { - logger = new Logger(this); - } - - #endregion - #region Population - /// - public override bool SetRemover(string field) + /// + /// Set remover from a value + /// + /// Key for the remover to be set + public bool SetRemover(string field) { // If the key is null or empty, return false if (string.IsNullOrWhiteSpace(field)) diff --git a/SabreTools.Filtering/Filter.cs b/SabreTools.Filtering/Filter.cs index 0127d34e..1b73d4ab 100644 --- a/SabreTools.Filtering/Filter.cs +++ b/SabreTools.Filtering/Filter.cs @@ -1,5 +1,11 @@ using System; +using System.Collections.Generic; +using System.Linq; +using SabreTools.Core; +using SabreTools.Core.Tools; +using SabreTools.DatFiles; +using SabreTools.DatItems; using SabreTools.Logging; namespace SabreTools.Filtering @@ -7,7 +13,7 @@ namespace SabreTools.Filtering /// /// Represents the filtering operations that need to be performed on a set of items, usually a DAT /// - public abstract class Filter + public class Filter { #region Constants @@ -37,6 +43,20 @@ namespace SabreTools.Filtering #endregion + #endregion + + #region Fields + + /// + /// Filter for DatItem fields + /// + public DatItemFilter DatItemFilter { get; set; } + + /// + /// Filter for Machine fields + /// + public MachineFilter MachineFilter { get; set; } + #endregion #region Logging @@ -60,7 +80,50 @@ namespace SabreTools.Filtering #endregion - #region Filter Population + #region Population + + /// + /// Populate the filters objects using a set of key:value filters + /// + /// List of key:value where ~key/!key is negated + public void PopulateFiltersFromList(List filters) + { + // Instantiate the filters, if necessary + MachineFilter ??= new MachineFilter(); + DatItemFilter ??= new DatItemFilter(); + + // If the list is null or empty, just return + if (filters == null || filters.Count == 0) + return; + + foreach (string filterPair in filters) + { + (string field, string value, bool negate) = ProcessFilterPair(filterPair); + + // If we don't even have a possible filter pair + if (field == null && value == null) + continue; + + // Machine fields + MachineField machineField = field.AsMachineField(); + if (machineField != MachineField.NULL) + { + MachineFilter.SetFilter(machineField, value, negate); + continue; + } + + // DatItem fields + DatItemField datItemField = field.AsDatItemField(); + if (datItemField != DatItemField.NULL) + { + DatItemFilter.SetFilter(datItemField, value, negate); + continue; + } + + // If we didn't match anything, log an error + logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names."); + } + } /// /// Split the parts of a filter statement @@ -294,7 +357,98 @@ namespace SabreTools.Filtering #endregion - #region Filter Running + #region Running + + /// + /// Apply a set of Filters on the DatFile + /// + /// Current DatFile object to run operations on + /// True if entire machines are considered, false otherwise (default) + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if the DatFile was filtered, false on error + public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false) + { + // If we have null filters, return false + if (MachineFilter == null || DatItemFilter == null) + return false; + + // If we're filtering per machine, bucket by machine first + if (perMachine) + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); + + try + { + // Loop over every key in the dictionary + List keys = datFile.Items.Keys.ToList(); + foreach (string key in keys) + { + // For every item in the current key + bool machinePass = true; + List items = datFile.Items[key]; + foreach (DatItem item in items) + { + // If we have a null item, we can't pass it + if (item == null) + continue; + + // If the item is already filtered out, we skip + if (item.Remove) + continue; + + // If the rom doesn't pass the filter, mark for removal + if (!PassesFilters(item)) + { + item.Remove = true; + + // If we're in machine mode, set and break + if (perMachine) + { + machinePass = false; + break; + } + } + } + + // If we didn't pass and we're in machine mode, set all items as remove + if (perMachine && !machinePass) + { + foreach (DatItem item in items) + { + item.Remove = true; + } + } + + // Assign back for caution + datFile.Items[key] = items; + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + /// + /// Check to see if a DatItem passes the filters + /// + /// DatItem to check + /// True if the item passed the filter, false otherwise + internal bool PassesFilters(DatItem datItem) + { + // Null item means it will never pass + if (datItem == null) + return false; + + // Filter on Machine fields + if (!MachineFilter.PassesFilters(datItem.Machine)) + return false; + + // Filter on DatItem fields + return DatItemFilter.PassesFilters(datItem); + } /// /// Determines if a value passes a bool? filter diff --git a/SabreTools.Filtering/Remover.cs b/SabreTools.Filtering/Remover.cs index f5b8a9bc..920fef76 100644 --- a/SabreTools.Filtering/Remover.cs +++ b/SabreTools.Filtering/Remover.cs @@ -1,3 +1,9 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +using SabreTools.Core; +using SabreTools.DatFiles; +using SabreTools.DatItems; using SabreTools.Logging; namespace SabreTools.Filtering @@ -5,8 +11,22 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a set of items, usually a DAT /// - public abstract class Remover + public class Remover { + #region Fields + + /// + /// DatItemRemover to remove fields from DatHeaders + /// + public DatHeaderRemover DatHeaderRemover { get; set; } + + /// + /// DatItemRemover to remove fields from DatItems + /// + public DatItemRemover DatItemRemover { get; set; } + + #endregion + #region Logging /// @@ -28,13 +48,78 @@ namespace SabreTools.Filtering #endregion - #region Remover Population + #region Population /// - /// Set remover from a value + /// Populate the exclusion objects using a set of field names /// - /// Key for the remover to be set - public abstract bool SetRemover(string field); + /// List of field names + public void PopulateExclusionsFromList(List fields) + { + // Instantiate the removers, if necessary + DatHeaderRemover ??= new DatHeaderRemover(); + DatItemRemover ??= new DatItemRemover(); + + // If the list is null or empty, just return + if (fields == null || fields.Count == 0) + return; + + foreach (string field in fields) + { + // If we don't even have a possible field name + if (field == null) + continue; + + // DatHeader fields + if (DatHeaderRemover.SetRemover(field)) + continue; + + // Machine and DatItem fields + if (DatItemRemover.SetRemover(field)) + continue; + + // If we didn't match anything, log an error + logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names."); + } + } + + #endregion + + #region Running + + /// + /// Remove fields from a DatFile + /// + /// Current DatFile object to run operations on + public void ApplyRemovals(DatFile datFile) + { + // If the removers don't exist, we can't use it + if (DatHeaderRemover == null && DatItemRemover == null) + return; + + // Output the logging statement + logger.User("Removing filtered fields"); + + // Remove DatHeader fields + if (DatHeaderRemover != null) + DatHeaderRemover.RemoveFields(datFile.Header); + + // Remove DatItem and Machine fields + if (DatItemRemover != null) + { + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + for (int j = 0; j < items.Count; j++) + { + DatItemRemover.RemoveFields(items[j]); + } + + datFile.Items.Remove(key); + datFile.Items.AddRange(key, items); + }); + } + } #endregion } diff --git a/SabreTools.Test/Filtering/FilteringTests.cs b/SabreTools.Test/Filtering/FilteringTests.cs index 98280d2b..6bf5e48a 100644 --- a/SabreTools.Test/Filtering/FilteringTests.cs +++ b/SabreTools.Test/Filtering/FilteringTests.cs @@ -11,60 +11,60 @@ namespace SabreTools.Test.Filtering [Fact] public void PassesFiltersDatItemFilterPass() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "item.name:foo" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "item.name:foo" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.True(actual); } [Fact] public void PassesFiltersDatItemFilterFail() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "item.name:bar" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "item.name:bar" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.False(actual); } [Fact] public void PassesFiltersMachineFilterPass() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "machine.name:bar" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "machine.name:bar" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.True(actual); } [Fact] public void PassesFiltersMachineFilterFail() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "machine.name:foo" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "machine.name:foo" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.False(actual); } diff --git a/SabreTools.Test/Filtering/PopulationTests.cs b/SabreTools.Test/Filtering/PopulationTests.cs index 5dd50748..3802e5e1 100644 --- a/SabreTools.Test/Filtering/PopulationTests.cs +++ b/SabreTools.Test/Filtering/PopulationTests.cs @@ -13,14 +13,14 @@ namespace SabreTools.Test.Filtering // Setup the list List exclusions = null; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -29,14 +29,14 @@ namespace SabreTools.Test.Filtering // Setup the list List exclusions = new List(); - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -48,14 +48,14 @@ namespace SabreTools.Test.Filtering "header.datname", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Single(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Single(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -67,14 +67,14 @@ namespace SabreTools.Test.Filtering "machine.name", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Single(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Single(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -86,14 +86,14 @@ namespace SabreTools.Test.Filtering "item.name", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Single(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Single(remover.DatItemRemover.DatItemFields); } [Fact] @@ -102,13 +102,13 @@ namespace SabreTools.Test.Filtering // Setup the list List filters = null; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.NotNull(cleaner.DatItemFilter); + Assert.NotNull(filter.MachineFilter); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -117,13 +117,13 @@ namespace SabreTools.Test.Filtering // Setup the list List filters = new List(); - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.NotNull(cleaner.DatItemFilter); + Assert.NotNull(filter.MachineFilter); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -136,14 +136,14 @@ namespace SabreTools.Test.Filtering "!machine.name:bar", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.Contains("foo", cleaner.MachineFilter.Name.PositiveSet); - Assert.Contains("bar", cleaner.MachineFilter.Name.NegativeSet); - Assert.NotNull(cleaner.DatItemFilter); + Assert.Contains("foo", filter.MachineFilter.Name.PositiveSet); + Assert.Contains("bar", filter.MachineFilter.Name.NegativeSet); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -156,14 +156,14 @@ namespace SabreTools.Test.Filtering "!item.name:bar" }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.Contains("foo", cleaner.DatItemFilter.Name.PositiveSet); - Assert.Contains("bar", cleaner.DatItemFilter.Name.NegativeSet); + Assert.NotNull(filter.MachineFilter); + Assert.Contains("foo", filter.DatItemFilter.Name.PositiveSet); + Assert.Contains("bar", filter.DatItemFilter.Name.NegativeSet); } } } \ No newline at end of file diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index f06b8a3b..c7c83b08 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -1725,6 +1725,11 @@ Some special strings that can be used: /// protected ExtraIni Extras { get; set; } + /// + /// Preconfigured Filter + /// + protected Filter Filter { get; set; } + /// /// Pre-configured DatHeader /// @@ -1735,6 +1740,11 @@ Some special strings that can be used: /// protected string OutputDir { get; set; } + /// + /// Pre-configured Remover + /// + protected Remover Remover { get; set; } + /// /// Pre-configured Splitter /// @@ -1804,8 +1814,10 @@ Some special strings that can be used: // Generic feature flags Cleaner = GetCleaner(features); Extras = GetExtras(features); + Filter = GetFilter(features); Header = GetDatHeader(features); OutputDir = GetString(features, OutputDirStringValue); + Remover = GetRemover(features); Splitter = GetSplitter(features); // Set threading flag, if necessary @@ -2011,9 +2023,6 @@ Some special strings that can be used: { Cleaner cleaner = new Cleaner() { - DatItemFilter = new DatItemFilter(), - MachineFilter = new MachineFilter(), - Clean = GetBoolean(features, CleanValue), DedupeRoms = GetDedupeType(features), DescriptionAsName = GetBoolean(features, DescriptionAsNameValue), @@ -2028,17 +2037,6 @@ Some special strings that can be used: Trim = GetBoolean(features, TrimValue), }; - // Populate field exclusions - List exclusionFields = GetList(features, ExcludeFieldListValue); - cleaner.PopulateExclusionsFromList(exclusionFields); - - // Populate filters - List filterPairs = GetList(features, FilterListValue); - cleaner.PopulateFiltersFromList(filterPairs); - - // Include 'of" in game filters - cleaner.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue); - return cleaner; } @@ -2121,6 +2119,42 @@ Some special strings that can be used: return extraIni; } + /// + /// Get Filter from feature list + /// + private Filter GetFilter(Dictionary features) + { + Filter filter = new Filter() + { + DatItemFilter = new DatItemFilter(), + MachineFilter = new MachineFilter(), + }; + + // Populate filters + List filterPairs = GetList(features, FilterListValue); + filter.PopulateFiltersFromList(filterPairs); + + // Include 'of" in game filters + filter.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue); + + return filter; + } + + /// + /// Get Remover from feature list + /// + private Remover GetRemover(Dictionary features) + { + Remover remover = new Remover(); + + // Populate field exclusions + List exclusionFields = GetList(features, ExcludeFieldListValue); + remover.PopulateExclusionsFromList(exclusionFields); + + return remover; + } + + /// /// Get Splitter from feature list /// diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index 4be2eed3..1ca4e5ed 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -159,8 +159,8 @@ Reset the internal state: reset();"; // TODO: We might not want to remove higher order hashes in the future // TODO: We might not want to remove dates in the future - Cleaner dfdCleaner = new Cleaner(); - dfdCleaner.PopulateExclusionsFromList(new List + Remover dfdRemover = new Remover(); + dfdRemover.PopulateExclusionsFromList(new List { "DatItem.SHA256", "DatItem.SHA384", @@ -168,7 +168,7 @@ Reset the internal state: reset();"; "DatItem.SpamSum", "DatItem.Date", }); - dfdCleaner.ApplyCleaning(datFile); + dfdRemover.ApplyRemovals(datFile); break; @@ -213,18 +213,18 @@ Reset the internal state: reset();"; } // Create cleaner to run filters from - Cleaner filterCleaner = new Cleaner + Filter filter = new Filter { MachineFilter = new MachineFilter(), DatItemFilter = new DatItemFilter(), }; // Set the possible filters - filterCleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); - filterCleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); + filter.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); + filter.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); // Apply the filters blindly - filterCleaner.ApplyFilters(datFile, filterPerMachine.Value); + filter.ApplyFilters(datFile, filterPerMachine.Value); // Cleanup after the filter // TODO: We might not want to remove immediately @@ -353,9 +353,9 @@ Reset the internal state: reset();"; } // Run the removal functionality - Cleaner remCleaner = new Cleaner(); - remCleaner.PopulateExclusionsFromList(command.Arguments); - remCleaner.RemoveFieldsFromItems(datFile); + Remover remover = new Remover(); + remover.PopulateExclusionsFromList(command.Arguments); + remover.ApplyRemovals(datFile); break; diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index 538788e6..51eab831 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -64,7 +64,7 @@ namespace SabreTools.Features // Apply the specialized field removals to the cleaner if (!addFileDates) - Cleaner.PopulateExclusionsFromList(new List { "DatItem.Date" }); + Remover.PopulateExclusionsFromList(new List { "DatItem.Date" }); // Create a new DATFromDir object and process the inputs DatFile basedat = DatFile.Create(Header); @@ -96,8 +96,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, false); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Write out the file Writer.Write(datdata, OutputDir); diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index 88df3ae7..48703b2a 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -158,8 +158,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datFile); Splitter.ApplySplitting(datFile, false); - Cleaner.ApplyFilters(datFile); + Filter.ApplyFilters(datFile); Cleaner.ApplyCleaning(datFile); + Remover.ApplyRemovals(datFile); // Get the correct output path string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue)); @@ -196,8 +197,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(userInputDat); Splitter.ApplySplitting(userInputDat, false); - Cleaner.ApplyFilters(userInputDat); + Filter.ApplyFilters(userInputDat); Cleaner.ApplyCleaning(userInputDat); + Remover.ApplyRemovals(userInputDat); // Output only DatItems that are duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) @@ -288,8 +290,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); Splitter.ApplySplitting(repDat, false); - Cleaner.ApplyFilters(repDat); + Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); + Remover.ApplyRemovals(repDat); // Now replace the fields from the base DatFile DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue)); @@ -313,8 +316,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); Splitter.ApplySplitting(repDat, false); - Cleaner.ApplyFilters(repDat); + Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); + Remover.ApplyRemovals(repDat); // Now replace the fields from the base DatFile DatFileTool.BaseReplace( diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 67390be2..9d47b4e1 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -63,8 +63,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, true); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; @@ -110,8 +111,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, true); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;