claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remover code to own class

Browse Source 
This was the last of the major bits of code that was technically distinct from Cleaning. This means that each of the bits of functionality that used to all be in Cleaner are split out into their approrpriate classes.
This commit is contained in:
Matt Nadareski
2021-02-01 12:11:32 -08:00
parent 69010dea7f
commit a82b7ccab8
13 changed files with 405 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
RombaSharp/Features/Dir2Dat.cs
View File

@@ -56,15 +56,15 @@ namespace RombaSharp.Features
datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name; datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
datfile.Header.Description = description; datfile.Header.Description = description;
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive); DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive);
Cleaner cleaner = new Cleaner(); Remover remover = new Remover();
cleaner.PopulateExclusionsFromList(new List<string> remover.PopulateExclusionsFromList(new List<string>
{ {
"DatItem.SHA256", "DatItem.SHA256",
"DatItem.SHA384", "DatItem.SHA384",
"DatItem.SHA512", "DatItem.SHA512",
"DatItem.SpamSum", "DatItem.SpamSum",
}); });
cleaner.ApplyCleaning(datfile); remover.ApplyRemovals(datfile);
Writer.Write(datfile, outdat); Writer.Write(datfile, outdat);
} }
} }

275
SabreTools.Filtering/Cleaner.cs
View File

@@ -8,7 +8,6 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
using SabreTools.Core; using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles; using SabreTools.DatFiles;
using SabreTools.DatItems; using SabreTools.DatItems;
using SabreTools.Logging; using SabreTools.Logging;
@@ -22,35 +21,7 @@ namespace SabreTools.Filtering
public class Cleaner public class Cleaner
{ {
#region Exclusion Fields #region Fields
/// <summary>
/// DatItemRemover to remove fields from DatHeaders
/// </summary>
public DatHeaderRemover DatHeaderRemover { get; set; }
/// <summary>
/// DatItemRemover to remove fields from DatItems
/// </summary>
public DatItemRemover DatItemRemover { get; set; }
#endregion
#region Filter Fields
/// <summary>
/// Filter for DatItem fields
/// </summary>
public DatItemFilter DatItemFilter { get; set; }
/// <summary>
/// Filter for Machine fields
/// </summary>
public MachineFilter MachineFilter { get; set; }
#endregion
#region Flag Fields
/// <summary> /// <summary>
/// Clean all names to WoD standards /// Clean all names to WoD standards
@@ -123,87 +94,7 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Population #region Running
/// <summary>
/// Populate the exclusion objects using a set of field names
/// </summary>
/// <param name="fields">List of field names</param>
public void PopulateExclusionsFromList(List<string> fields)
{
// Instantiate the removers, if necessary
DatHeaderRemover ??= new DatHeaderRemover();
DatItemRemover ??= new DatItemRemover();
// If the list is null or empty, just return
if (fields == null || fields.Count == 0)
return;
foreach (string field in fields)
{
// If we don't even have a possible field name
if (field == null)
continue;
// DatHeader fields
if (DatHeaderRemover.SetRemover(field))
continue;
// Machine and DatItem fields
if (DatItemRemover.SetRemover(field))
continue;
// If we didn't match anything, log an error
logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names.");
}
}
/// <summary>
/// Populate the filters objects using a set of key:value filters
/// </summary>
/// <param name="filters">List of key:value where ~key/!key is negated</param>
public void PopulateFiltersFromList(List<string> filters)
{
// Instantiate the filters, if necessary
MachineFilter ??= new MachineFilter();
DatItemFilter ??= new DatItemFilter();
// If the list is null or empty, just return
if (filters == null || filters.Count == 0)
return;
foreach (string filterPair in filters)
{
(string field, string value, bool negate) = ProcessFilterPair(filterPair);
// If we don't even have a possible filter pair
if (field == null && value == null)
continue;
// Machine fields
MachineField machineField = field.AsMachineField();
if (machineField != MachineField.NULL)
{
MachineFilter.SetFilter(machineField, value, negate);
continue;
}
// DatItem fields
DatItemField datItemField = field.AsDatItemField();
if (datItemField != DatItemField.NULL)
{
DatItemFilter.SetFilter(datItemField, value, negate);
continue;
}
// If we didn't match anything, log an error
logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names.");
}
}
#endregion
#region Cleaning
/// <summary> /// <summary>
/// Apply cleaning methods to the DatFile /// Apply cleaning methods to the DatFile
@@ -240,9 +131,6 @@ namespace SabreTools.Filtering
if (OneRomPerGame == true) if (OneRomPerGame == true)
SetOneRomPerGame(datFile); SetOneRomPerGame(datFile);
// If we are removing fields, do that now
RemoveFieldsFromItems(datFile);
// Remove all marked items // Remove all marked items
datFile.Items.ClearMarked(); datFile.Items.ClearMarked();
@@ -691,164 +579,5 @@ namespace SabreTools.Filtering
} }
#endregion #endregion
#region Filtering
/// <summary>
/// Apply a set of Filters on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="perMachine">True if entire machines are considered, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was filtered, false on error</returns>
public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false)
{
// If we have null filters, return false
if (MachineFilter == null || DatItemFilter == null)
return false;
// If we're filtering per machine, bucket by machine first
if (perMachine)
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
try
{
// Loop over every key in the dictionary
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
bool machinePass = true;
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't pass it
if (item == null)
continue;
// If the item is already filtered out, we skip
if (item.Remove)
continue;
// If the rom doesn't pass the filter, mark for removal
if (!PassesFilters(item))
{
item.Remove = true;
// If we're in machine mode, set and break
if (perMachine)
{
machinePass = false;
break;
}
}
}
// If we didn't pass and we're in machine mode, set all items as remove
if (perMachine && !machinePass)
{
foreach (DatItem item in items)
{
item.Remove = true;
}
}
// Assign back for caution
datFile.Items[key] = items;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Check to see if a DatItem passes the filters
/// </summary>
/// <param name="datItem">DatItem to check</param>
/// <returns>True if the item passed the filter, false otherwise</returns>
internal bool PassesFilters(DatItem datItem)
{
// Null item means it will never pass
if (datItem == null)
return false;
// Filter on Machine fields
if (!MachineFilter.PassesFilters(datItem.Machine))
return false;
// Filter on DatItem fields
return DatItemFilter.PassesFilters(datItem);
}
/// <summary>
/// Split the parts of a filter statement
/// </summary>
/// <param name="filter">key:value where ~key/!key is negated</param>
private (string field, string value, bool negate) ProcessFilterPair(string filter)
{
// If we don't even have a possible filter pair
if (!filter.Contains(":"))
{
logger.Warning($"'{filter}` is not a valid filter string. Valid filter strings are of the form 'key:value'. Please refer to README.1ST or the help feature for more details.");
return (null, null, false);
}
string filterTrimmed = filter.Trim('"', ' ', '\t');
bool negate = filterTrimmed.StartsWith("!")
|| filterTrimmed.StartsWith("~")
|| filterTrimmed.StartsWith("not-");
filterTrimmed = filterTrimmed.TrimStart('!', '~');
filterTrimmed = filterTrimmed.StartsWith("not-") ? filterTrimmed[4..] : filterTrimmed;
string filterFieldString = filterTrimmed.Split(':')[0].ToLowerInvariant().Trim('"', ' ', '\t');
string filterValue = filterTrimmed[(filterFieldString.Length + 1)..].Trim('"', ' ', '\t');
return (filterFieldString, filterValue, negate);
}
#endregion
#region Removal
/// <summary>
/// Remove fields as per the header
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
public void RemoveFieldsFromItems(DatFile datFile)
{
// If the removers don't exist, we can't use it
if (DatHeaderRemover == null && DatItemRemover == null)
return;
// Output the logging statement
logger.User("Removing filtered fields");
// Remove DatHeader fields
if (DatHeaderRemover != null)
DatHeaderRemover.RemoveFields(datFile.Header);
// Remove DatItem and Machine fields
if (DatItemRemover != null)
{
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int j = 0; j < items.Count; j++)
{
DatItemRemover.RemoveFields(items[j]);
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, items);
});
}
}
#endregion
} }
} }

21
SabreTools.Filtering/DatHeaderRemover.cs
View File

@@ -10,7 +10,7 @@ namespace SabreTools.Filtering
/// <summary> /// <summary>
/// Represents the removal operations that need to be performed on a DatHeader /// Represents the removal operations that need to be performed on a DatHeader
/// </summary> /// </summary>
public class DatHeaderRemover : Remover public class DatHeaderRemover
{ {
#region Fields #region Fields
@@ -21,22 +21,13 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Constructors
/// <summary>
/// Constructor
/// </summary>
public DatHeaderRemover()
{
logger = new Logger(this);
}
#endregion
#region Population #region Population
/// <inheritdoc/> /// <summary>
public override bool SetRemover(string field) /// Set remover from a value
/// </summary>
/// <param name="field">Key for the remover to be set</param>
public bool SetRemover(string field)
{ {
// If the key is null or empty, return false // If the key is null or empty, return false
if (string.IsNullOrWhiteSpace(field)) if (string.IsNullOrWhiteSpace(field))

21
SabreTools.Filtering/DatItemRemover.cs
View File

@@ -10,7 +10,7 @@ namespace SabreTools.Filtering
/// <summary> /// <summary>
/// Represents the removal operations that need to be performed on a set of items, usually a DAT /// Represents the removal operations that need to be performed on a set of items, usually a DAT
/// </summary> /// </summary>
public class DatItemRemover : Remover public class DatItemRemover
{ {
#region Fields #region Fields
@@ -26,22 +26,13 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Constructors
/// <summary>
/// Constructor
/// </summary>
public DatItemRemover()
{
logger = new Logger(this);
}
#endregion
#region Population #region Population
/// <inheritdoc/> /// <summary>
public override bool SetRemover(string field) /// Set remover from a value
/// </summary>
/// <param name="field">Key for the remover to be set</param>
public bool SetRemover(string field)
{ {
// If the key is null or empty, return false // If the key is null or empty, return false
if (string.IsNullOrWhiteSpace(field)) if (string.IsNullOrWhiteSpace(field))

160
SabreTools.Filtering/Filter.cs
View File

@@ -1,5 +1,11 @@
using System; using System;
using System.Collections.Generic;
using System.Linq;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Logging; using SabreTools.Logging;
namespace SabreTools.Filtering namespace SabreTools.Filtering
@@ -7,7 +13,7 @@ namespace SabreTools.Filtering
/// <summary> /// <summary>
/// Represents the filtering operations that need to be performed on a set of items, usually a DAT /// Represents the filtering operations that need to be performed on a set of items, usually a DAT
/// </summary> /// </summary>
public abstract class Filter public class Filter
{ {
#region Constants #region Constants
@@ -37,6 +43,20 @@ namespace SabreTools.Filtering
#endregion #endregion
#endregion
#region Fields
/// <summary>
/// Filter for DatItem fields
/// </summary>
public DatItemFilter DatItemFilter { get; set; }
/// <summary>
/// Filter for Machine fields
/// </summary>
public MachineFilter MachineFilter { get; set; }
#endregion #endregion
#region Logging #region Logging
@@ -60,7 +80,50 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Filter Population #region Population
/// <summary>
/// Populate the filters objects using a set of key:value filters
/// </summary>
/// <param name="filters">List of key:value where ~key/!key is negated</param>
public void PopulateFiltersFromList(List<string> filters)
{
// Instantiate the filters, if necessary
MachineFilter ??= new MachineFilter();
DatItemFilter ??= new DatItemFilter();
// If the list is null or empty, just return
if (filters == null || filters.Count == 0)
return;
foreach (string filterPair in filters)
{
(string field, string value, bool negate) = ProcessFilterPair(filterPair);
// If we don't even have a possible filter pair
if (field == null && value == null)
continue;
// Machine fields
MachineField machineField = field.AsMachineField();
if (machineField != MachineField.NULL)
{
MachineFilter.SetFilter(machineField, value, negate);
continue;
}
// DatItem fields
DatItemField datItemField = field.AsDatItemField();
if (datItemField != DatItemField.NULL)
{
DatItemFilter.SetFilter(datItemField, value, negate);
continue;
}
// If we didn't match anything, log an error
logger.Warning($"The value {field} did not match any filterable field names. Please check the wiki for more details on supported field names.");
}
}
/// <summary> /// <summary>
/// Split the parts of a filter statement /// Split the parts of a filter statement
@@ -294,7 +357,98 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Filter Running #region Running
/// <summary>
/// Apply a set of Filters on the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="perMachine">True if entire machines are considered, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DatFile was filtered, false on error</returns>
public bool ApplyFilters(DatFile datFile, bool perMachine = false, bool throwOnError = false)
{
// If we have null filters, return false
if (MachineFilter == null || DatItemFilter == null)
return false;
// If we're filtering per machine, bucket by machine first
if (perMachine)
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
try
{
// Loop over every key in the dictionary
List<string> keys = datFile.Items.Keys.ToList();
foreach (string key in keys)
{
// For every item in the current key
bool machinePass = true;
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If we have a null item, we can't pass it
if (item == null)
continue;
// If the item is already filtered out, we skip
if (item.Remove)
continue;
// If the rom doesn't pass the filter, mark for removal
if (!PassesFilters(item))
{
item.Remove = true;
// If we're in machine mode, set and break
if (perMachine)
{
machinePass = false;
break;
}
}
}
// If we didn't pass and we're in machine mode, set all items as remove
if (perMachine && !machinePass)
{
foreach (DatItem item in items)
{
item.Remove = true;
}
}
// Assign back for caution
datFile.Items[key] = items;
}
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
return true;
}
/// <summary>
/// Check to see if a DatItem passes the filters
/// </summary>
/// <param name="datItem">DatItem to check</param>
/// <returns>True if the item passed the filter, false otherwise</returns>
internal bool PassesFilters(DatItem datItem)
{
// Null item means it will never pass
if (datItem == null)
return false;
// Filter on Machine fields
if (!MachineFilter.PassesFilters(datItem.Machine))
return false;
// Filter on DatItem fields
return DatItemFilter.PassesFilters(datItem);
}
/// <summary> /// <summary>
/// Determines if a value passes a bool? filter /// Determines if a value passes a bool? filter

95
SabreTools.Filtering/Remover.cs
View File

@@ -1,3 +1,9 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Logging; using SabreTools.Logging;
namespace SabreTools.Filtering namespace SabreTools.Filtering
@@ -5,8 +11,22 @@ namespace SabreTools.Filtering
/// <summary> /// <summary>
/// Represents the removal operations that need to be performed on a set of items, usually a DAT /// Represents the removal operations that need to be performed on a set of items, usually a DAT
/// </summary> /// </summary>
public abstract class Remover public class Remover
{ {
#region Fields
/// <summary>
/// DatItemRemover to remove fields from DatHeaders
/// </summary>
public DatHeaderRemover DatHeaderRemover { get; set; }
/// <summary>
/// DatItemRemover to remove fields from DatItems
/// </summary>
public DatItemRemover DatItemRemover { get; set; }
#endregion
#region Logging #region Logging
/// <summary> /// <summary>
@@ -28,13 +48,78 @@ namespace SabreTools.Filtering
#endregion #endregion
#region Remover Population #region Population
/// <summary> /// <summary>
/// Set remover from a value /// Populate the exclusion objects using a set of field names
/// </summary> /// </summary>
/// <param name="field">Key for the remover to be set</param> /// <param name="fields">List of field names</param>
public abstract bool SetRemover(string field); public void PopulateExclusionsFromList(List<string> fields)
{
// Instantiate the removers, if necessary
DatHeaderRemover ??= new DatHeaderRemover();
DatItemRemover ??= new DatItemRemover();
// If the list is null or empty, just return
if (fields == null || fields.Count == 0)
return;
foreach (string field in fields)
{
// If we don't even have a possible field name
if (field == null)
continue;
// DatHeader fields
if (DatHeaderRemover.SetRemover(field))
continue;
// Machine and DatItem fields
if (DatItemRemover.SetRemover(field))
continue;
// If we didn't match anything, log an error
logger.Warning($"The value {field} did not match any known field names. Please check the wiki for more details on supported field names.");
}
}
#endregion
#region Running
/// <summary>
/// Remove fields from a DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
public void ApplyRemovals(DatFile datFile)
{
// If the removers don't exist, we can't use it
if (DatHeaderRemover == null && DatItemRemover == null)
return;
// Output the logging statement
logger.User("Removing filtered fields");
// Remove DatHeader fields
if (DatHeaderRemover != null)
DatHeaderRemover.RemoveFields(datFile.Header);
// Remove DatItem and Machine fields
if (DatItemRemover != null)
{
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
for (int j = 0; j < items.Count; j++)
{
DatItemRemover.RemoveFields(items[j]);
}
datFile.Items.Remove(key);
datFile.Items.AddRange(key, items);
});
}
}
#endregion #endregion
} }

32
SabreTools.Test/Filtering/FilteringTests.cs
View File

@@ -11,60 +11,60 @@ namespace SabreTools.Test.Filtering
[Fact] [Fact]
public void PassesFiltersDatItemFilterPass() public void PassesFiltersDatItemFilterPass()
{ {
// Setup cleaner // Setup filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(new List<string> { "item.name:foo" }); filter.PopulateFiltersFromList(new List<string> { "item.name:foo" });
// Setup DatItem // Setup DatItem
var datItem = CreateDatItem(); var datItem = CreateDatItem();
// Run filters // Run filters
bool actual = cleaner.PassesFilters(datItem); bool actual = filter.PassesFilters(datItem);
Assert.True(actual); Assert.True(actual);
} }
[Fact] [Fact]
public void PassesFiltersDatItemFilterFail() public void PassesFiltersDatItemFilterFail()
{ {
// Setup cleaner // Setup filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(new List<string> { "item.name:bar" }); filter.PopulateFiltersFromList(new List<string> { "item.name:bar" });
// Setup DatItem // Setup DatItem
var datItem = CreateDatItem(); var datItem = CreateDatItem();
// Run filters // Run filters
bool actual = cleaner.PassesFilters(datItem); bool actual = filter.PassesFilters(datItem);
Assert.False(actual); Assert.False(actual);
} }
[Fact] [Fact]
public void PassesFiltersMachineFilterPass() public void PassesFiltersMachineFilterPass()
{ {
// Setup cleaner // Setup filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(new List<string> { "machine.name:bar" }); filter.PopulateFiltersFromList(new List<string> { "machine.name:bar" });
// Setup DatItem // Setup DatItem
var datItem = CreateDatItem(); var datItem = CreateDatItem();
// Run filters // Run filters
bool actual = cleaner.PassesFilters(datItem); bool actual = filter.PassesFilters(datItem);
Assert.True(actual); Assert.True(actual);
} }
[Fact] [Fact]
public void PassesFiltersMachineFilterFail() public void PassesFiltersMachineFilterFail()
{ {
// Setup cleaner // Setup filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(new List<string> { "machine.name:foo" }); filter.PopulateFiltersFromList(new List<string> { "machine.name:foo" });
// Setup DatItem // Setup DatItem
var datItem = CreateDatItem(); var datItem = CreateDatItem();
// Run filters // Run filters
bool actual = cleaner.PassesFilters(datItem); bool actual = filter.PassesFilters(datItem);
Assert.False(actual); Assert.False(actual);
} }

104
SabreTools.Test/Filtering/PopulationTests.cs
View File

@@ -13,14 +13,14 @@ namespace SabreTools.Test.Filtering
// Setup the list // Setup the list
List<string> exclusions = null; List<string> exclusions = null;
// Setup the cleaner // Setup the remover
var cleaner = new Cleaner(); var remover = new Remover();
cleaner.PopulateExclusionsFromList(exclusions); remover.PopulateExclusionsFromList(exclusions);
// Check the exclusion lists // Check the exclusion lists
Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); Assert.Empty(remover.DatHeaderRemover.DatHeaderFields);
Assert.Empty(cleaner.DatItemRemover.MachineFields); Assert.Empty(remover.DatItemRemover.MachineFields);
Assert.Empty(cleaner.DatItemRemover.DatItemFields); Assert.Empty(remover.DatItemRemover.DatItemFields);
} }
[Fact] [Fact]
@@ -29,14 +29,14 @@ namespace SabreTools.Test.Filtering
// Setup the list // Setup the list
List<string> exclusions = new List<string>(); List<string> exclusions = new List<string>();
// Setup the cleaner // Setup the remover
var cleaner = new Cleaner(); var remover = new Remover();
cleaner.PopulateExclusionsFromList(exclusions); remover.PopulateExclusionsFromList(exclusions);
// Check the exclusion lists // Check the exclusion lists
Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); Assert.Empty(remover.DatHeaderRemover.DatHeaderFields);
Assert.Empty(cleaner.DatItemRemover.MachineFields); Assert.Empty(remover.DatItemRemover.MachineFields);
Assert.Empty(cleaner.DatItemRemover.DatItemFields); Assert.Empty(remover.DatItemRemover.DatItemFields);
} }
[Fact] [Fact]
@@ -48,14 +48,14 @@ namespace SabreTools.Test.Filtering
"header.datname", "header.datname",
}; };
// Setup the cleaner // Setup the remover
var cleaner = new Cleaner(); var remover = new Remover();
cleaner.PopulateExclusionsFromList(exclusions); remover.PopulateExclusionsFromList(exclusions);
// Check the exclusion lists // Check the exclusion lists
Assert.Single(cleaner.DatHeaderRemover.DatHeaderFields); Assert.Single(remover.DatHeaderRemover.DatHeaderFields);
Assert.Empty(cleaner.DatItemRemover.MachineFields); Assert.Empty(remover.DatItemRemover.MachineFields);
Assert.Empty(cleaner.DatItemRemover.DatItemFields); Assert.Empty(remover.DatItemRemover.DatItemFields);
} }
[Fact] [Fact]
@@ -67,14 +67,14 @@ namespace SabreTools.Test.Filtering
"machine.name", "machine.name",
}; };
// Setup the cleaner // Setup the remover
var cleaner = new Cleaner(); var remover = new Remover();
cleaner.PopulateExclusionsFromList(exclusions); remover.PopulateExclusionsFromList(exclusions);
// Check the exclusion lists // Check the exclusion lists
Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); Assert.Empty(remover.DatHeaderRemover.DatHeaderFields);
Assert.Single(cleaner.DatItemRemover.MachineFields); Assert.Single(remover.DatItemRemover.MachineFields);
Assert.Empty(cleaner.DatItemRemover.DatItemFields); Assert.Empty(remover.DatItemRemover.DatItemFields);
} }
[Fact] [Fact]
@@ -86,14 +86,14 @@ namespace SabreTools.Test.Filtering
"item.name", "item.name",
}; };
// Setup the cleaner // Setup the remover
var cleaner = new Cleaner(); var remover = new Remover();
cleaner.PopulateExclusionsFromList(exclusions); remover.PopulateExclusionsFromList(exclusions);
// Check the exclusion lists // Check the exclusion lists
Assert.Empty(cleaner.DatHeaderRemover.DatHeaderFields); Assert.Empty(remover.DatHeaderRemover.DatHeaderFields);
Assert.Empty(cleaner.DatItemRemover.MachineFields); Assert.Empty(remover.DatItemRemover.MachineFields);
Assert.Single(cleaner.DatItemRemover.DatItemFields); Assert.Single(remover.DatItemRemover.DatItemFields);
} }
[Fact] [Fact]
@@ -102,13 +102,13 @@ namespace SabreTools.Test.Filtering
// Setup the list // Setup the list
List<string> filters = null; List<string> filters = null;
// Setup the cleaner // Setup the filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(filters); filter.PopulateFiltersFromList(filters);
// Check the filters // Check the filters
Assert.NotNull(cleaner.MachineFilter); Assert.NotNull(filter.MachineFilter);
Assert.NotNull(cleaner.DatItemFilter); Assert.NotNull(filter.DatItemFilter);
} }
[Fact] [Fact]
@@ -117,13 +117,13 @@ namespace SabreTools.Test.Filtering
// Setup the list // Setup the list
List<string> filters = new List<string>(); List<string> filters = new List<string>();
// Setup the cleaner // Setup the filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(filters); filter.PopulateFiltersFromList(filters);
// Check the filters // Check the filters
Assert.NotNull(cleaner.MachineFilter); Assert.NotNull(filter.MachineFilter);
Assert.NotNull(cleaner.DatItemFilter); Assert.NotNull(filter.DatItemFilter);
} }
[Fact] [Fact]
@@ -136,14 +136,14 @@ namespace SabreTools.Test.Filtering
"!machine.name:bar", "!machine.name:bar",
}; };
// Setup the cleaner // Setup the filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(filters); filter.PopulateFiltersFromList(filters);
// Check the filters // Check the filters
Assert.Contains("foo", cleaner.MachineFilter.Name.PositiveSet); Assert.Contains("foo", filter.MachineFilter.Name.PositiveSet);
Assert.Contains("bar", cleaner.MachineFilter.Name.NegativeSet); Assert.Contains("bar", filter.MachineFilter.Name.NegativeSet);
Assert.NotNull(cleaner.DatItemFilter); Assert.NotNull(filter.DatItemFilter);
} }
[Fact] [Fact]
@@ -156,14 +156,14 @@ namespace SabreTools.Test.Filtering
"!item.name:bar" "!item.name:bar"
}; };
// Setup the cleaner // Setup the filter
var cleaner = new Cleaner(); var filter = new Filter();
cleaner.PopulateFiltersFromList(filters); filter.PopulateFiltersFromList(filters);
// Check the filters // Check the filters
Assert.NotNull(cleaner.MachineFilter); Assert.NotNull(filter.MachineFilter);
Assert.Contains("foo", cleaner.DatItemFilter.Name.PositiveSet); Assert.Contains("foo", filter.DatItemFilter.Name.PositiveSet);
Assert.Contains("bar", cleaner.DatItemFilter.Name.NegativeSet); Assert.Contains("bar", filter.DatItemFilter.Name.NegativeSet);
} }
} }
} }

62
SabreTools/Features/BaseFeature.cs
View File

@@ -1725,6 +1725,11 @@ Some special strings that can be used:
/// </summary> /// </summary>
protected ExtraIni Extras { get; set; } protected ExtraIni Extras { get; set; }
/// <summary>
/// Preconfigured Filter
/// </summary>
protected Filter Filter { get; set; }
/// <summary> /// <summary>
/// Pre-configured DatHeader /// Pre-configured DatHeader
/// </summary> /// </summary>
@@ -1735,6 +1740,11 @@ Some special strings that can be used:
/// </summary> /// </summary>
protected string OutputDir { get; set; } protected string OutputDir { get; set; }
/// <summary>
/// Pre-configured Remover
/// </summary>
protected Remover Remover { get; set; }
/// <summary> /// <summary>
/// Pre-configured Splitter /// Pre-configured Splitter
/// </summary> /// </summary>
@@ -1804,8 +1814,10 @@ Some special strings that can be used:
// Generic feature flags // Generic feature flags
Cleaner = GetCleaner(features); Cleaner = GetCleaner(features);
Extras = GetExtras(features); Extras = GetExtras(features);
Filter = GetFilter(features);
Header = GetDatHeader(features); Header = GetDatHeader(features);
OutputDir = GetString(features, OutputDirStringValue); OutputDir = GetString(features, OutputDirStringValue);
Remover = GetRemover(features);
Splitter = GetSplitter(features); Splitter = GetSplitter(features);
// Set threading flag, if necessary // Set threading flag, if necessary
@@ -2011,9 +2023,6 @@ Some special strings that can be used:
{ {
Cleaner cleaner = new Cleaner() Cleaner cleaner = new Cleaner()
{ {
DatItemFilter = new DatItemFilter(),
MachineFilter = new MachineFilter(),
Clean = GetBoolean(features, CleanValue), Clean = GetBoolean(features, CleanValue),
DedupeRoms = GetDedupeType(features), DedupeRoms = GetDedupeType(features),
DescriptionAsName = GetBoolean(features, DescriptionAsNameValue), DescriptionAsName = GetBoolean(features, DescriptionAsNameValue),
@@ -2028,17 +2037,6 @@ Some special strings that can be used:
Trim = GetBoolean(features, TrimValue), Trim = GetBoolean(features, TrimValue),
}; };
// Populate field exclusions
List<string> exclusionFields = GetList(features, ExcludeFieldListValue);
cleaner.PopulateExclusionsFromList(exclusionFields);
// Populate filters
List<string> filterPairs = GetList(features, FilterListValue);
cleaner.PopulateFiltersFromList(filterPairs);
// Include 'of" in game filters
cleaner.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue);
return cleaner; return cleaner;
} }
@@ -2121,6 +2119,42 @@ Some special strings that can be used:
return extraIni; return extraIni;
} }
/// <summary>
/// Get Filter from feature list
/// </summary>
private Filter GetFilter(Dictionary<string, Feature> features)
{
Filter filter = new Filter()
{
DatItemFilter = new DatItemFilter(),
MachineFilter = new MachineFilter(),
};
// Populate filters
List<string> filterPairs = GetList(features, FilterListValue);
filter.PopulateFiltersFromList(filterPairs);
// Include 'of" in game filters
filter.MachineFilter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue);
return filter;
}
/// <summary>
/// Get Remover from feature list
/// </summary>
private Remover GetRemover(Dictionary<string, Feature> features)
{
Remover remover = new Remover();
// Populate field exclusions
List<string> exclusionFields = GetList(features, ExcludeFieldListValue);
remover.PopulateExclusionsFromList(exclusionFields);
return remover;
}
/// <summary> /// <summary>
/// Get Splitter from feature list /// Get Splitter from feature list
/// </summary> /// </summary>

20
SabreTools/Features/Batch.cs
View File

@@ -159,8 +159,8 @@ Reset the internal state: reset();";
// TODO: We might not want to remove higher order hashes in the future // TODO: We might not want to remove higher order hashes in the future
// TODO: We might not want to remove dates in the future // TODO: We might not want to remove dates in the future
Cleaner dfdCleaner = new Cleaner(); Remover dfdRemover = new Remover();
dfdCleaner.PopulateExclusionsFromList(new List<string> dfdRemover.PopulateExclusionsFromList(new List<string>
{ {
"DatItem.SHA256", "DatItem.SHA256",
"DatItem.SHA384", "DatItem.SHA384",
@@ -168,7 +168,7 @@ Reset the internal state: reset();";
"DatItem.SpamSum", "DatItem.SpamSum",
"DatItem.Date", "DatItem.Date",
}); });
dfdCleaner.ApplyCleaning(datFile); dfdRemover.ApplyRemovals(datFile);
break; break;
@@ -213,18 +213,18 @@ Reset the internal state: reset();";
} }
// Create cleaner to run filters from // Create cleaner to run filters from
Cleaner filterCleaner = new Cleaner Filter filter = new Filter
{ {
MachineFilter = new MachineFilter(), MachineFilter = new MachineFilter(),
DatItemFilter = new DatItemFilter(), DatItemFilter = new DatItemFilter(),
}; };
// Set the possible filters // Set the possible filters
filterCleaner.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value); filter.MachineFilter.SetFilter(filterMachineField, filterValue, filterRemove.Value);
filterCleaner.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value); filter.DatItemFilter.SetFilter(filterDatItemField, filterValue, filterRemove.Value);
// Apply the filters blindly // Apply the filters blindly
filterCleaner.ApplyFilters(datFile, filterPerMachine.Value); filter.ApplyFilters(datFile, filterPerMachine.Value);
// Cleanup after the filter // Cleanup after the filter
// TODO: We might not want to remove immediately // TODO: We might not want to remove immediately
@@ -353,9 +353,9 @@ Reset the internal state: reset();";
} }
// Run the removal functionality // Run the removal functionality
Cleaner remCleaner = new Cleaner(); Remover remover = new Remover();
remCleaner.PopulateExclusionsFromList(command.Arguments); remover.PopulateExclusionsFromList(command.Arguments);
remCleaner.RemoveFieldsFromItems(datFile); remover.ApplyRemovals(datFile);
break; break;

5
SabreTools/Features/DatFromDir.cs
View File

@@ -64,7 +64,7 @@ namespace SabreTools.Features
// Apply the specialized field removals to the cleaner // Apply the specialized field removals to the cleaner
if (!addFileDates) if (!addFileDates)
Cleaner.PopulateExclusionsFromList(new List<string> { "DatItem.Date" }); Remover.PopulateExclusionsFromList(new List<string> { "DatItem.Date" });
// Create a new DATFromDir object and process the inputs // Create a new DATFromDir object and process the inputs
DatFile basedat = DatFile.Create(Header); DatFile basedat = DatFile.Create(Header);
@@ -96,8 +96,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(datdata); Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, false); Splitter.ApplySplitting(datdata, false);
Cleaner.ApplyFilters(datdata); Filter.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata); Cleaner.ApplyCleaning(datdata);
Remover.ApplyRemovals(datdata);
// Write out the file // Write out the file
Writer.Write(datdata, OutputDir); Writer.Write(datdata, OutputDir);

12
SabreTools/Features/Update.cs
View File

@@ -158,8 +158,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(datFile); Extras.ApplyExtras(datFile);
Splitter.ApplySplitting(datFile, false); Splitter.ApplySplitting(datFile, false);
Cleaner.ApplyFilters(datFile); Filter.ApplyFilters(datFile);
Cleaner.ApplyCleaning(datFile); Cleaner.ApplyCleaning(datFile);
Remover.ApplyRemovals(datFile);
// Get the correct output path // Get the correct output path
string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue)); string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
@@ -196,8 +197,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(userInputDat); Extras.ApplyExtras(userInputDat);
Splitter.ApplySplitting(userInputDat, false); Splitter.ApplySplitting(userInputDat, false);
Cleaner.ApplyFilters(userInputDat); Filter.ApplyFilters(userInputDat);
Cleaner.ApplyCleaning(userInputDat); Cleaner.ApplyCleaning(userInputDat);
Remover.ApplyRemovals(userInputDat);
// Output only DatItems that are duplicated across inputs // Output only DatItems that are duplicated across inputs
if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) if (updateMode.HasFlag(UpdateMode.DiffDupesOnly))
@@ -288,8 +290,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(repDat); Extras.ApplyExtras(repDat);
Splitter.ApplySplitting(repDat, false); Splitter.ApplySplitting(repDat, false);
Cleaner.ApplyFilters(repDat); Filter.ApplyFilters(repDat);
Cleaner.ApplyCleaning(repDat); Cleaner.ApplyCleaning(repDat);
Remover.ApplyRemovals(repDat);
// Now replace the fields from the base DatFile // Now replace the fields from the base DatFile
DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue)); DatFileTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue));
@@ -313,8 +316,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(repDat); Extras.ApplyExtras(repDat);
Splitter.ApplySplitting(repDat, false); Splitter.ApplySplitting(repDat, false);
Cleaner.ApplyFilters(repDat); Filter.ApplyFilters(repDat);
Cleaner.ApplyCleaning(repDat); Cleaner.ApplyCleaning(repDat);
Remover.ApplyRemovals(repDat);
// Now replace the fields from the base DatFile // Now replace the fields from the base DatFile
DatFileTool.BaseReplace( DatFileTool.BaseReplace(

6
SabreTools/Features/Verify.cs
View File

@@ -63,8 +63,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(datdata); Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, true); Splitter.ApplySplitting(datdata, true);
Cleaner.ApplyFilters(datdata); Filter.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata); Cleaner.ApplyCleaning(datdata);
Remover.ApplyRemovals(datdata);
// Set depot information // Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
@@ -110,8 +111,9 @@ namespace SabreTools.Features
// Perform additional processing steps // Perform additional processing steps
Extras.ApplyExtras(datdata); Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, true); Splitter.ApplySplitting(datdata, true);
Cleaner.ApplyFilters(datdata); Filter.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata); Cleaner.ApplyCleaning(datdata);
Remover.ApplyRemovals(datdata);
// Set depot information // Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;