diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index 7836a4b1..a7faa733 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -56,15 +56,15 @@ namespace RombaSharp.Features datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name; datfile.Header.Description = description; DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive); - Cleaner cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(new List + Remover remover = new Remover(); + remover.PopulateExclusionsFromList(new List { "DatItem.SHA256", "DatItem.SHA384", "DatItem.SHA512", "DatItem.SpamSum", }); - cleaner.ApplyCleaning(datfile); + remover.ApplyRemovals(datfile); Writer.Write(datfile, outdat); } } diff --git a/SabreTools.Filtering/Cleaner.cs b/SabreTools.Filtering/Cleaner.cs index b1635780..f182ffca 100644 --- a/SabreTools.Filtering/Cleaner.cs +++ b/SabreTools.Filtering/Cleaner.cs @@ -8,7 +8,6 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using SabreTools.Core; -using SabreTools.Core.Tools; using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.Logging; @@ -22,35 +21,7 @@ namespace SabreTools.Filtering public class Cleaner { - #region Exclusion Fields - - /// - /// DatItemRemover to remove fields from DatHeaders - /// - public DatHeaderRemover DatHeaderRemover { get; set; } - - /// - /// DatItemRemover to remove fields from DatItems - /// - public DatItemRemover DatItemRemover { get; set; } - - #endregion - - #region Filter Fields - - /// - /// Filter for DatItem fields - /// - public DatItemFilter DatItemFilter { get; set; } - - /// - /// Filter for Machine fields - /// - public MachineFilter MachineFilter { get; set; } - - #endregion - - #region Flag Fields + #region Fields /// /// Clean all names to WoD standards @@ -123,87 +94,7 @@ namespace SabreTools.Filtering #endregion - #region Population - - /// - /// Populate the exclusion objects using a set of field names - /// - /// List of field names - public void PopulateExclusionsFromList(List fields) - { - // Instantiate the removers, if necessary - DatHeaderRemover ??= new DatHeaderRemover(); - DatItemRemover ??= new DatItemRemover(); - - // If the list is null or empty, just return - if (fields == null || fields.Count == 0) - return; - - foreach (string field in fields) - { - // If we don't even have a possible field name - if (field == null) - continue; - - // DatHeader fields - if (DatHeaderRemover.SetRemover(field)) - continue; - - // Machine and DatItem fields - if (DatItemRemover.SetRemover(field)) - continue; - - // If we didn't match anything, log an error - logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names."); - } - } - - /// - /// Populate the filters objects using a set of key:value filters - /// - /// List of key:value where ~key/!key is negated - public void PopulateFiltersFromList(List filters) - { - // Instantiate the filters, if necessary - MachineFilter ??= new MachineFilter(); - DatItemFilter ??= new DatItemFilter(); - - // If the list is null or empty, just return - if (filters == null || filters.Count == 0) - return; - - foreach (string filterPair in filters) - { - (string field, string value, bool negate) = ProcessFilterPair(filterPair); - - // If we don't even have a possible filter pair - if (field == null && value == null) - continue; - - // Machine fields - MachineField machineField = field.AsMachineField(); - if (machineField != MachineField.NULL) - { - MachineFilter.SetFilter(machineField, value, negate); - continue; - } - - // DatItem fields - DatItemField datItemField = field.AsDatItemField(); - if (datItemField != DatItemField.NULL) - { - DatItemFilter.SetFilter(datItemField, value, negate); - continue; - } - - // If we didn't match anything, log an error - logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names."); - } - } - - #endregion - - #region Cleaning + #region Running /// /// Apply cleaning methods to the DatFile @@ -240,9 +131,6 @@ namespace SabreTools.Filtering if (OneRomPerGame == true) SetOneRomPerGame(datFile); - // If we are removing fields, do that now - RemoveFieldsFromItems(datFile); - // Remove all marked items datFile.Items.ClearMarked(); @@ -691,164 +579,5 @@ namespace SabreTools.Filtering } #endregion - - #region Filtering - - /// - /// Apply a set of Filters on the DatFile - /// - /// Current DatFile object to run operations on - /// True if entire machines are considered, false otherwise (default) - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the DatFile was filtered, false on error - public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false) - { - // If we have null filters, return false - if (MachineFilter == null || DatItemFilter == null) - return false; - - // If we're filtering per machine, bucket by machine first - if (perMachine) - datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); - - try - { - // Loop over every key in the dictionary - List keys = datFile.Items.Keys.ToList(); - foreach (string key in keys) - { - // For every item in the current key - bool machinePass = true; - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If we have a null item, we can't pass it - if (item == null) - continue; - - // If the item is already filtered out, we skip - if (item.Remove) - continue; - - // If the rom doesn't pass the filter, mark for removal - if (!PassesFilters(item)) - { - item.Remove = true; - - // If we're in machine mode, set and break - if (perMachine) - { - machinePass = false; - break; - } - } - } - - // If we didn't pass and we're in machine mode, set all items as remove - if (perMachine && !machinePass) - { - foreach (DatItem item in items) - { - item.Remove = true; - } - } - - // Assign back for caution - datFile.Items[key] = items; - } - } - catch (Exception ex) when (!throwOnError) - { - logger.Error(ex); - return false; - } - - return true; - } - - /// - /// Check to see if a DatItem passes the filters - /// - /// DatItem to check - /// True if the item passed the filter, false otherwise - internal bool PassesFilters(DatItem datItem) - { - // Null item means it will never pass - if (datItem == null) - return false; - - // Filter on Machine fields - if (!MachineFilter.PassesFilters(datItem.Machine)) - return false; - - // Filter on DatItem fields - return DatItemFilter.PassesFilters(datItem); - } - - /// - /// Split the parts of a filter statement - /// - /// key:value where ~key/!key is negated - private (string field, string value, bool negate) ProcessFilterPair(string filter) - { - // If we don't even have a possible filter pair - if (!filter.Contains(":")) - { - logger.Warning($"'{filter}` is not a valid filter string. Valid filter strings are of the form 'key:value'. Please refer to README.1ST or the help feature for more details."); - return (null, null, false); - } - - string filterTrimmed = filter.Trim('"', ' ', '\t'); - bool negate = filterTrimmed.StartsWith("!") - || filterTrimmed.StartsWith("~") - || filterTrimmed.StartsWith("not-"); - filterTrimmed = filterTrimmed.TrimStart('!', '~'); - filterTrimmed = filterTrimmed.StartsWith("not-") ? filterTrimmed[4..] : filterTrimmed; - - string filterFieldString = filterTrimmed.Split(':')[0].ToLowerInvariant().Trim('"', ' ', '\t'); - string filterValue = filterTrimmed[(filterFieldString.Length + 1)..].Trim('"', ' ', '\t'); - - return (filterFieldString, filterValue, negate); - } - - #endregion - - #region Removal - - /// - /// Remove fields as per the header - /// - /// Current DatFile object to run operations on - public void RemoveFieldsFromItems(DatFile datFile) - { - // If the removers don't exist, we can't use it - if (DatHeaderRemover == null && DatItemRemover == null) - return; - - // Output the logging statement - logger.User("Removing filtered fields"); - - // Remove DatHeader fields - if (DatHeaderRemover != null) - DatHeaderRemover.RemoveFields(datFile.Header); - - // Remove DatItem and Machine fields - if (DatItemRemover != null) - { - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int j = 0; j < items.Count; j++) - { - DatItemRemover.RemoveFields(items[j]); - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, items); - }); - } - } - - #endregion } } diff --git a/SabreTools.Filtering/DatHeaderRemover.cs b/SabreTools.Filtering/DatHeaderRemover.cs index 18bf1a02..1d10e0da 100644 --- a/SabreTools.Filtering/DatHeaderRemover.cs +++ b/SabreTools.Filtering/DatHeaderRemover.cs @@ -10,7 +10,7 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a DatHeader /// - public class DatHeaderRemover : Remover + public class DatHeaderRemover { #region Fields @@ -21,22 +21,13 @@ namespace SabreTools.Filtering #endregion - #region Constructors - - /// - /// Constructor - /// - public DatHeaderRemover() - { - logger = new Logger(this); - } - - #endregion - #region Population - /// - public override bool SetRemover(string field) + /// + /// Set remover from a value + /// + /// Key for the remover to be set + public bool SetRemover(string field) { // If the key is null or empty, return false if (string.IsNullOrWhiteSpace(field)) diff --git a/SabreTools.Filtering/DatItemRemover.cs b/SabreTools.Filtering/DatItemRemover.cs index c6341093..17b2e80a 100644 --- a/SabreTools.Filtering/DatItemRemover.cs +++ b/SabreTools.Filtering/DatItemRemover.cs @@ -10,7 +10,7 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a set of items, usually a DAT /// - public class DatItemRemover : Remover + public class DatItemRemover { #region Fields @@ -26,22 +26,13 @@ namespace SabreTools.Filtering #endregion - #region Constructors - - /// - /// Constructor - /// - public DatItemRemover() - { - logger = new Logger(this); - } - - #endregion - #region Population - /// - public override bool SetRemover(string field) + /// + /// Set remover from a value + /// + /// Key for the remover to be set + public bool SetRemover(string field) { // If the key is null or empty, return false if (string.IsNullOrWhiteSpace(field)) diff --git a/SabreTools.Filtering/Filter.cs b/SabreTools.Filtering/Filter.cs index 0127d34e..1b73d4ab 100644 --- a/SabreTools.Filtering/Filter.cs +++ b/SabreTools.Filtering/Filter.cs @@ -1,5 +1,11 @@ using System; +using System.Collections.Generic; +using System.Linq; +using SabreTools.Core; +using SabreTools.Core.Tools; +using SabreTools.DatFiles; +using SabreTools.DatItems; using SabreTools.Logging; namespace SabreTools.Filtering @@ -7,7 +13,7 @@ namespace SabreTools.Filtering /// /// Represents the filtering operations that need to be performed on a set of items, usually a DAT /// - public abstract class Filter + public class Filter { #region Constants @@ -37,6 +43,20 @@ namespace SabreTools.Filtering #endregion + #endregion + + #region Fields + + /// + /// Filter for DatItem fields + /// + public DatItemFilter DatItemFilter { get; set; } + + /// + /// Filter for Machine fields + /// + public MachineFilter MachineFilter { get; set; } + #endregion #region Logging @@ -60,7 +80,50 @@ namespace SabreTools.Filtering #endregion - #region Filter Population + #region Population + + /// + /// Populate the filters objects using a set of key:value filters + /// + /// List of key:value where ~key/!key is negated + public void PopulateFiltersFromList(List filters) + { + // Instantiate the filters, if necessary + MachineFilter ??= new MachineFilter(); + DatItemFilter ??= new DatItemFilter(); + + // If the list is null or empty, just return + if (filters == null || filters.Count == 0) + return; + + foreach (string filterPair in filters) + { + (string field, string value, bool negate) = ProcessFilterPair(filterPair); + + // If we don't even have a possible filter pair + if (field == null && value == null) + continue; + + // Machine fields + MachineField machineField = field.AsMachineField(); + if (machineField != MachineField.NULL) + { + MachineFilter.SetFilter(machineField, value, negate); + continue; + } + + // DatItem fields + DatItemField datItemField = field.AsDatItemField(); + if (datItemField != DatItemField.NULL) + { + DatItemFilter.SetFilter(datItemField, value, negate); + continue; + } + + // If we didn't match anything, log an error + logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names."); + } + } /// /// Split the parts of a filter statement @@ -294,7 +357,98 @@ namespace SabreTools.Filtering #endregion - #region Filter Running + #region Running + + /// + /// Apply a set of Filters on the DatFile + /// + /// Current DatFile object to run operations on + /// True if entire machines are considered, false otherwise (default) + /// True if the error that is thrown should be thrown back to the caller, false otherwise + /// True if the DatFile was filtered, false on error + public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false) + { + // If we have null filters, return false + if (MachineFilter == null || DatItemFilter == null) + return false; + + // If we're filtering per machine, bucket by machine first + if (perMachine) + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None); + + try + { + // Loop over every key in the dictionary + List keys = datFile.Items.Keys.ToList(); + foreach (string key in keys) + { + // For every item in the current key + bool machinePass = true; + List items = datFile.Items[key]; + foreach (DatItem item in items) + { + // If we have a null item, we can't pass it + if (item == null) + continue; + + // If the item is already filtered out, we skip + if (item.Remove) + continue; + + // If the rom doesn't pass the filter, mark for removal + if (!PassesFilters(item)) + { + item.Remove = true; + + // If we're in machine mode, set and break + if (perMachine) + { + machinePass = false; + break; + } + } + } + + // If we didn't pass and we're in machine mode, set all items as remove + if (perMachine && !machinePass) + { + foreach (DatItem item in items) + { + item.Remove = true; + } + } + + // Assign back for caution + datFile.Items[key] = items; + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + /// + /// Check to see if a DatItem passes the filters + /// + /// DatItem to check + /// True if the item passed the filter, false otherwise + internal bool PassesFilters(DatItem datItem) + { + // Null item means it will never pass + if (datItem == null) + return false; + + // Filter on Machine fields + if (!MachineFilter.PassesFilters(datItem.Machine)) + return false; + + // Filter on DatItem fields + return DatItemFilter.PassesFilters(datItem); + } /// /// Determines if a value passes a bool? filter diff --git a/SabreTools.Filtering/Remover.cs b/SabreTools.Filtering/Remover.cs index f5b8a9bc..920fef76 100644 --- a/SabreTools.Filtering/Remover.cs +++ b/SabreTools.Filtering/Remover.cs @@ -1,3 +1,9 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +using SabreTools.Core; +using SabreTools.DatFiles; +using SabreTools.DatItems; using SabreTools.Logging; namespace SabreTools.Filtering @@ -5,8 +11,22 @@ namespace SabreTools.Filtering /// /// Represents the removal operations that need to be performed on a set of items, usually a DAT /// - public abstract class Remover + public class Remover { + #region Fields + + /// + /// DatItemRemover to remove fields from DatHeaders + /// + public DatHeaderRemover DatHeaderRemover { get; set; } + + /// + /// DatItemRemover to remove fields from DatItems + /// + public DatItemRemover DatItemRemover { get; set; } + + #endregion + #region Logging /// @@ -28,13 +48,78 @@ namespace SabreTools.Filtering #endregion - #region Remover Population + #region Population /// - /// Set remover from a value + /// Populate the exclusion objects using a set of field names /// - /// Key for the remover to be set - public abstract bool SetRemover(string field); + /// List of field names + public void PopulateExclusionsFromList(List fields) + { + // Instantiate the removers, if necessary + DatHeaderRemover ??= new DatHeaderRemover(); + DatItemRemover ??= new DatItemRemover(); + + // If the list is null or empty, just return + if (fields == null || fields.Count == 0) + return; + + foreach (string field in fields) + { + // If we don't even have a possible field name + if (field == null) + continue; + + // DatHeader fields + if (DatHeaderRemover.SetRemover(field)) + continue; + + // Machine and DatItem fields + if (DatItemRemover.SetRemover(field)) + continue; + + // If we didn't match anything, log an error + logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names."); + } + } + + #endregion + + #region Running + + /// + /// Remove fields from a DatFile + /// + /// Current DatFile object to run operations on + public void ApplyRemovals(DatFile datFile) + { + // If the removers don't exist, we can't use it + if (DatHeaderRemover == null && DatItemRemover == null) + return; + + // Output the logging statement + logger.User("Removing filtered fields"); + + // Remove DatHeader fields + if (DatHeaderRemover != null) + DatHeaderRemover.RemoveFields(datFile.Header); + + // Remove DatItem and Machine fields + if (DatItemRemover != null) + { + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => + { + List items = datFile.Items[key]; + for (int j = 0; j < items.Count; j++) + { + DatItemRemover.RemoveFields(items[j]); + } + + datFile.Items.Remove(key); + datFile.Items.AddRange(key, items); + }); + } + } #endregion } diff --git a/SabreTools.Test/Filtering/FilteringTests.cs b/SabreTools.Test/Filtering/FilteringTests.cs index 98280d2b..6bf5e48a 100644 --- a/SabreTools.Test/Filtering/FilteringTests.cs +++ b/SabreTools.Test/Filtering/FilteringTests.cs @@ -11,60 +11,60 @@ namespace SabreTools.Test.Filtering [Fact] public void PassesFiltersDatItemFilterPass() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "item.name:foo" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "item.name:foo" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.True(actual); } [Fact] public void PassesFiltersDatItemFilterFail() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "item.name:bar" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "item.name:bar" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.False(actual); } [Fact] public void PassesFiltersMachineFilterPass() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "machine.name:bar" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "machine.name:bar" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.True(actual); } [Fact] public void PassesFiltersMachineFilterFail() { - // Setup cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(new List { "machine.name:foo" }); + // Setup filter + var filter = new Filter(); + filter.PopulateFiltersFromList(new List { "machine.name:foo" }); // Setup DatItem var datItem = CreateDatItem(); // Run filters - bool actual = cleaner.PassesFilters(datItem); + bool actual = filter.PassesFilters(datItem); Assert.False(actual); } diff --git a/SabreTools.Test/Filtering/PopulationTests.cs b/SabreTools.Test/Filtering/PopulationTests.cs index 5dd50748..3802e5e1 100644 --- a/SabreTools.Test/Filtering/PopulationTests.cs +++ b/SabreTools.Test/Filtering/PopulationTests.cs @@ -13,14 +13,14 @@ namespace SabreTools.Test.Filtering // Setup the list List exclusions = null; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -29,14 +29,14 @@ namespace SabreTools.Test.Filtering // Setup the list List exclusions = new List(); - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -48,14 +48,14 @@ namespace SabreTools.Test.Filtering "header.datname", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Single(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Single(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -67,14 +67,14 @@ namespace SabreTools.Test.Filtering "machine.name", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Single(cleaner.DatItemRemover.MachineFields); - Assert.Empty(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Single(remover.DatItemRemover.MachineFields); + Assert.Empty(remover.DatItemRemover.DatItemFields); } [Fact] @@ -86,14 +86,14 @@ namespace SabreTools.Test.Filtering "item.name", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateExclusionsFromList(exclusions); + // Setup the remover + var remover = new Remover(); + remover.PopulateExclusionsFromList(exclusions); // Check the exclusion lists - Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); - Assert.Empty(cleaner.DatItemRemover.MachineFields); - Assert.Single(cleaner.DatItemRemover.DatItemFields); + Assert.Empty(remover.DatHeaderRemover.DatHeaderFields); + Assert.Empty(remover.DatItemRemover.MachineFields); + Assert.Single(remover.DatItemRemover.DatItemFields); } [Fact] @@ -102,13 +102,13 @@ namespace SabreTools.Test.Filtering // Setup the list List filters = null; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.NotNull(cleaner.DatItemFilter); + Assert.NotNull(filter.MachineFilter); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -117,13 +117,13 @@ namespace SabreTools.Test.Filtering // Setup the list List filters = new List(); - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.NotNull(cleaner.DatItemFilter); + Assert.NotNull(filter.MachineFilter); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -136,14 +136,14 @@ namespace SabreTools.Test.Filtering "!machine.name:bar", }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.Contains("foo", cleaner.MachineFilter.Name.PositiveSet); - Assert.Contains("bar", cleaner.MachineFilter.Name.NegativeSet); - Assert.NotNull(cleaner.DatItemFilter); + Assert.Contains("foo", filter.MachineFilter.Name.PositiveSet); + Assert.Contains("bar", filter.MachineFilter.Name.NegativeSet); + Assert.NotNull(filter.DatItemFilter); } [Fact] @@ -156,14 +156,14 @@ namespace SabreTools.Test.Filtering "!item.name:bar" }; - // Setup the cleaner - var cleaner = new Cleaner(); - cleaner.PopulateFiltersFromList(filters); + // Setup the filter + var filter = new Filter(); + filter.PopulateFiltersFromList(filters); // Check the filters - Assert.NotNull(cleaner.MachineFilter); - Assert.Contains("foo", cleaner.DatItemFilter.Name.PositiveSet); - Assert.Contains("bar", cleaner.DatItemFilter.Name.NegativeSet); + Assert.NotNull(filter.MachineFilter); + Assert.Contains("foo", filter.DatItemFilter.Name.PositiveSet); + Assert.Contains("bar", filter.DatItemFilter.Name.NegativeSet); } } } \ No newline at end of file diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index f06b8a3b..c7c83b08 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -1725,6 +1725,11 @@ Some special strings that can be used: /// protected ExtraIni Extras { get; set; } + /// + /// Preconfigured Filter + /// + protected Filter Filter { get; set; } + /// /// Pre-configured DatHeader /// @@ -1735,6 +1740,11 @@ Some special strings that can be used: /// protected string OutputDir { get; set; } + /// + /// Pre-configured Remover + /// + protected Remover Remover { get; set; } + /// /// Pre-configured Splitter /// @@ -1804,8 +1814,10 @@ Some special strings that can be used: // Generic feature flags Cleaner = GetCleaner(features); Extras = GetExtras(features); + Filter = GetFilter(features); Header = GetDatHeader(features); OutputDir = GetString(features, OutputDirStringValue); + Remover = GetRemover(features); Splitter = GetSplitter(features); // Set threading flag, if necessary @@ -2011,9 +2023,6 @@ Some special strings that can be used: { Cleaner cleaner = new Cleaner() { - DatItemFilter = new DatItemFilter(), - MachineFilter = new MachineFilter(), - Clean = GetBoolean(features, CleanValue), DedupeRoms = GetDedupeType(features), DescriptionAsName = GetBoolean(features, DescriptionAsNameValue), @@ -2028,17 +2037,6 @@ Some special strings that can be used: Trim = GetBoolean(features, TrimValue), }; - // Populate field exclusions - List exclusionFields = GetList(features, ExcludeFieldListValue); - cleaner.PopulateExclusionsFromList(exclusionFields); - - // Populate filters - List filterPairs = GetList(features, FilterListValue); - cleaner.PopulateFiltersFromList(filterPairs); - - // Include 'of" in game filters - cleaner.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue); - return cleaner; } @@ -2121,6 +2119,42 @@ Some special strings that can be used: return extraIni; } + /// + /// Get Filter from feature list + /// + private Filter GetFilter(Dictionary features) + { + Filter filter = new Filter() + { + DatItemFilter = new DatItemFilter(), + MachineFilter = new MachineFilter(), + }; + + // Populate filters + List filterPairs = GetList(features, FilterListValue); + filter.PopulateFiltersFromList(filterPairs); + + // Include 'of" in game filters + filter.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue); + + return filter; + } + + /// + /// Get Remover from feature list + /// + private Remover GetRemover(Dictionary features) + { + Remover remover = new Remover(); + + // Populate field exclusions + List exclusionFields = GetList(features, ExcludeFieldListValue); + remover.PopulateExclusionsFromList(exclusionFields); + + return remover; + } + + /// /// Get Splitter from feature list /// diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index 4be2eed3..1ca4e5ed 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -159,8 +159,8 @@ Reset the internal state: reset();"; // TODO: We might not want to remove higher order hashes in the future // TODO: We might not want to remove dates in the future - Cleaner dfdCleaner = new Cleaner(); - dfdCleaner.PopulateExclusionsFromList(new List + Remover dfdRemover = new Remover(); + dfdRemover.PopulateExclusionsFromList(new List { "DatItem.SHA256", "DatItem.SHA384", @@ -168,7 +168,7 @@ Reset the internal state: reset();"; "DatItem.SpamSum", "DatItem.Date", }); - dfdCleaner.ApplyCleaning(datFile); + dfdRemover.ApplyRemovals(datFile); break; @@ -213,18 +213,18 @@ Reset the internal state: reset();"; } // Create cleaner to run filters from - Cleaner filterCleaner = new Cleaner + Filter filter = new Filter { MachineFilter = new MachineFilter(), DatItemFilter = new DatItemFilter(), }; // Set the possible filters - filterCleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); - filterCleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); + filter.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); + filter.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); // Apply the filters blindly - filterCleaner.ApplyFilters(datFile, filterPerMachine.Value); + filter.ApplyFilters(datFile, filterPerMachine.Value); // Cleanup after the filter // TODO: We might not want to remove immediately @@ -353,9 +353,9 @@ Reset the internal state: reset();"; } // Run the removal functionality - Cleaner remCleaner = new Cleaner(); - remCleaner.PopulateExclusionsFromList(command.Arguments); - remCleaner.RemoveFieldsFromItems(datFile); + Remover remover = new Remover(); + remover.PopulateExclusionsFromList(command.Arguments); + remover.ApplyRemovals(datFile); break; diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index 538788e6..51eab831 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -64,7 +64,7 @@ namespace SabreTools.Features // Apply the specialized field removals to the cleaner if (!addFileDates) - Cleaner.PopulateExclusionsFromList(new List { "DatItem.Date" }); + Remover.PopulateExclusionsFromList(new List { "DatItem.Date" }); // Create a new DATFromDir object and process the inputs DatFile basedat = DatFile.Create(Header); @@ -96,8 +96,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, false); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Write out the file Writer.Write(datdata, OutputDir); diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index 88df3ae7..48703b2a 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -158,8 +158,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datFile); Splitter.ApplySplitting(datFile, false); - Cleaner.ApplyFilters(datFile); + Filter.ApplyFilters(datFile); Cleaner.ApplyCleaning(datFile); + Remover.ApplyRemovals(datFile); // Get the correct output path string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue)); @@ -196,8 +197,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(userInputDat); Splitter.ApplySplitting(userInputDat, false); - Cleaner.ApplyFilters(userInputDat); + Filter.ApplyFilters(userInputDat); Cleaner.ApplyCleaning(userInputDat); + Remover.ApplyRemovals(userInputDat); // Output only DatItems that are duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) @@ -288,8 +290,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); Splitter.ApplySplitting(repDat, false); - Cleaner.ApplyFilters(repDat); + Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); + Remover.ApplyRemovals(repDat); // Now replace the fields from the base DatFile DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue)); @@ -313,8 +316,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); Splitter.ApplySplitting(repDat, false); - Cleaner.ApplyFilters(repDat); + Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); + Remover.ApplyRemovals(repDat); // Now replace the fields from the base DatFile DatFileTool.BaseReplace( diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 67390be2..9d47b4e1 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -63,8 +63,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, true); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; @@ -110,8 +111,9 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, true); - Cleaner.ApplyFilters(datdata); + Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); + Remover.ApplyRemovals(datdata); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;