Create and use Cleaner

SabreTools.Library/DatFiles/DatFile.cs

@@ -695,15 +695,49 @@ namespace SabreTools.Library.DatFiles
 
         #endregion
 
-        // TODO: Is it possible for all non-cleaning operations to be non-destructive?
+        // TODO: Should the ApplyFilter method contain the splitting logic?
-        // If items are not flat out removed and just marked for removal instead, this
-        // could allow for rolling back removals and effectively "unfiltering" a DatFile
-        // without too much hassle. On write it would have to take into account these
-        // removed items, however. Similarly, this could lead to interesting issues
-        // with merging and splitting since removed items might end up causing empty
-        // machines on the other side
         #region Filtering
 
+        /// <summary>
+        /// Apply cleaning methods to the DatFile
+        /// </summary>
+        /// <param name="cleaner">Cleaner to use</param>
+        /// <returns>True if cleaning was successful, false on error</returns>
+        public bool ApplyCleaning(Cleaner cleaner)
+        {
+            // Perform item-level cleaning
+            CleanDatItems(cleaner);
+
+            // Process description to machine name
+            if (cleaner?.DescriptionAsName == true)
+                MachineDescriptionToName();
+
+            // If we are removing scene dates, do that now
+            if (cleaner?.SceneDateStrip == true)
+                StripSceneDatesFromItems();
+
+            // Run the one rom per game logic, if required
+            if (cleaner?.OneGamePerRegion == true)
+                OneGamePerRegion(cleaner.RegionList);
+
+            // Run the one rom per game logic, if required
+            if (cleaner?.OneRomPerGame == true)
+                OneRomPerGame();
+
+            // If we are removing fields, do that now
+            if (cleaner.ExcludeFields != null && cleaner.ExcludeFields.Any())
+                RemoveFieldsFromItems(cleaner.ExcludeFields);
+
+            // Remove all marked items
+            Items.ClearMarked();
+
+            // We remove any blanks, if we aren't supposed to have any
+            if (cleaner?.KeepEmptyGames == false)
+                Items.ClearEmpty();
+
+            return true;
+        }
+
         /// <summary>
         /// Apply a set of Extra INIs on the DatFile
         /// </summary>
@@ -773,15 +807,14 @@ namespace SabreTools.Library.DatFiles
         /// <param name="filter">Filter to use</param>
         /// <param name="useTags">True if DatFile tags override splitting, false otherwise</param>
         /// <returns>True if the DatFile was filtered, false on error</returns>
-        /// TODO: Re-evaluate what should be in here and see if we need to have a "Cleaner" class
         public bool ApplyFilter(Filter filter, bool useTags)
         {
+            // If we have a null filter, return false
+            if (filter == null)
+                return false;
+
             try
             {
-                // Process description to machine name
-                if (filter.DescriptionAsName)
-                    MachineDescriptionToName();
-
                 // If we are using tags from the DAT, set the proper input for split type unless overridden
                 if (useTags && filter.InternalSplit == MergingFlag.None)
                     filter.InternalSplit = Header.ForceMerging;
@@ -801,91 +834,14 @@ namespace SabreTools.Library.DatFiles
                         if (item == null)
                             continue;
 
-                        // If the rom passes the filter, include it
+                        // If the rom doesn't pass the filter, mark for removal
-                        if (item.PassesFilter(filter))
+                        if (!item.PassesFilter(filter))
-                        {
-                            // If we're stripping unicode characters, do so from all relevant things
-                            if (filter.RemoveUnicode)
-                            {
-                                item.Name = Sanitizer.RemoveUnicodeCharacters(item.Name);
-                                item.Machine.Name = Sanitizer.RemoveUnicodeCharacters(item.Machine.Name);
-                                item.Machine.Description = Sanitizer.RemoveUnicodeCharacters(item.Machine.Description);
-                            }
-
-                            // If we're in cleaning mode, do so from all relevant things
-                            if (filter.Clean)
-                            {
-                                item.Machine.Name = Sanitizer.CleanGameName(item.Machine.Name);
-                                item.Machine.Description = Sanitizer.CleanGameName(item.Machine.Description);
-                            }
-
-                            // If we are in single game mode, rename all games
-                            if (filter.Single)
-                                item.Machine.Name = "!";
-
-                            // If we are in NTFS trim mode, trim the game name
-                            if (filter.Trim)
-                            {
-                                // Windows max name length is 260
-                                int usableLength = 260 - item.Machine.Name.Length - filter.Root.Length;
-                                if (item.Name.Length > usableLength)
-                                {
-                                    string ext = Path.GetExtension(item.Name);
-                                    item.Name = item.Name.Substring(0, usableLength - ext.Length);
-                                    item.Name += ext;
-                                }
-                            }
-                        }
-
-                        // Otherwise mark for removal
-                        else
-                        {
                             item.Remove = true;
-                        }
                     }
 
                     // Assign back for caution
                     Items[key] = items;
                 }
-
-                // If we are removing scene dates, do that now
-                if (Header.SceneDateStrip)
-                    StripSceneDatesFromItems();
-
-                // Run the one rom per game logic, if required
-                if (Header.OneGamePerRegion)
-                    OneGamePerRegion();
-
-                // Run the one rom per game logic, if required
-                if (Header.OneRomPerGame)
-                    OneRomPerGame();
-
-                // If we are removing fields, do that now
-                if (Header.ExcludeFields != null && Header.ExcludeFields.Any())
-                    RemoveFieldsFromItems();
-
-                // Remove all marked items
-                Items.ClearMarked();
-
-                // We remove any blanks, if we aren't supposed to have any
-                if (!Header.KeepEmptyGames)
-                {
-                    List<string> possiblyEmptyKeys = Items.Keys.ToList();
-                    foreach (string key in possiblyEmptyKeys)
-                    {
-                        List<DatItem> items = Items[key];
-                        if (items == null || items.Count == 0)
-                        {
-                            Items.Remove(key);
-                            continue;
-                        }
-
-                        List<DatItem> newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList();
-
-                        Items.Remove(key);
-                        Items.AddRange(key, newitems);
-                    }
-                }
             }
             catch (Exception ex)
             {
@@ -929,6 +885,61 @@ namespace SabreTools.Library.DatFiles
             });
         }
 
+        /// <summary>
+        /// Clean individual items based on the current filter
+        /// </summary>
+        /// <param name="cleaner">Cleaner to use</param>
+        public void CleanDatItems(Cleaner cleaner)
+        {
+            List<string> keys = Items.Keys.ToList();
+            foreach (string key in keys)
+            {
+                // For every item in the current key
+                List<DatItem> items = Items[key];
+                foreach (DatItem item in items)
+                {
+                    // If we have a null item, we can't clean it it
+                    if (item == null)
+                        continue;
+
+                    // If we're stripping unicode characters, do so from all relevant things
+                    if (cleaner?.RemoveUnicode == true)
+                    {
+                        item.Name = Sanitizer.RemoveUnicodeCharacters(item.Name);
+                        item.Machine.Name = Sanitizer.RemoveUnicodeCharacters(item.Machine.Name);
+                        item.Machine.Description = Sanitizer.RemoveUnicodeCharacters(item.Machine.Description);
+                    }
+
+                    // If we're in cleaning mode, do so from all relevant things
+                    if (cleaner?.Clean == true)
+                    {
+                        item.Machine.Name = Sanitizer.CleanGameName(item.Machine.Name);
+                        item.Machine.Description = Sanitizer.CleanGameName(item.Machine.Description);
+                    }
+
+                    // If we are in single game mode, rename all games
+                    if (cleaner?.Single == true)
+                        item.Machine.Name = "!";
+
+                    // If we are in NTFS trim mode, trim the game name
+                    if (cleaner?.Trim == true)
+                    {
+                        // Windows max name length is 260
+                        int usableLength = 260 - item.Machine.Name.Length - (cleaner.Root?.Length ?? 0);
+                        if (item.Name.Length > usableLength)
+                        {
+                            string ext = Path.GetExtension(item.Name);
+                            item.Name = item.Name.Substring(0, usableLength - ext.Length);
+                            item.Name += ext;
+                        }
+                    }
+                }
+
+                // Assign back for caution
+                Items[key] = items;
+            }
+        }
+
         /// <summary>
         /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof
         /// </summary>
@@ -989,6 +1000,7 @@ namespace SabreTools.Library.DatFiles
         /// <summary>
         /// Filter a DAT using 1G1R logic given an ordered set of regions
         /// </summary>
+        /// <param name="regions">Ordered list of regions to use</param>
         /// <remarks>
         /// In the most technical sense, the way that the region list is being used does not
         /// confine its values to be just regions. Since it's essentially acting like a
@@ -999,8 +1011,12 @@ namespace SabreTools.Library.DatFiles
         /// to clone sets based on name, nor does it have the ability to match on the 
         /// Release DatItem type.
         /// </remarks>
-        public void OneGamePerRegion()
+        public void OneGamePerRegion(List<string> regions)
         {
+            // If we have null region list, make it empty
+            if (regions == null)
+                regions = new List<string>();
+
             // For sake of ease, the first thing we want to do is bucket by game
             Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true);
 
@@ -1038,11 +1054,6 @@ namespace SabreTools.Library.DatFiles
                 }
             }
 
-            // If we have null region list, make it empty
-            List<string> regions = Header.RegionList;
-            if (regions == null)
-                regions = new List<string>();
-
             // Once we have the full list of mappings, filter out games to keep
             foreach (string key in parents.Keys)
             {
@@ -1094,14 +1105,16 @@ namespace SabreTools.Library.DatFiles
         /// <summary>
         /// Remove fields as per the header
         /// </summary>
-        public void RemoveFieldsFromItems()
+        /// <param name="fields">List of fields to use</param>
+        public void RemoveFieldsFromItems(List<Field> fields)
         {
+            // If we have null field list, make it empty
+            if (fields == null)
+                fields = new List<Field>();
+
             // Output the logging statement
             Globals.Logger.User("Removing filtered fields");
 
-            // Get the array of fields from the header
-            List<Field> fields = Header.ExcludeFields;
-
             // Now process all of the roms
             Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
             {

SabreTools.Library/DatFiles/DatHeader.cs

@@ -267,42 +267,6 @@ namespace SabreTools.Library.DatFiles
 
         #region Filtering Fields
 
-        /// <summary>
-        /// Dictionary of fields in machine and items to exclude from writing
-        /// </summary>
-        [JsonIgnore]
-        public List<Field> ExcludeFields { get; set; } = new List<Field>();
-
-        /// <summary>
-        /// Enable "One Rom, One Region (1G1R)" mode
-        /// </summary>
-        [JsonIgnore]
-        public bool OneGamePerRegion { get; set; }
-
-        /// <summary>
-        /// Ordered list of regions for "One Rom, One Region (1G1R)" mode
-        /// </summary>
-        [JsonIgnore]
-        public List<string> RegionList { get; set; }
-
-        /// <summary>
-        /// Ensure each rom is in their own game
-        /// </summary>
-        [JsonIgnore]
-        public bool OneRomPerGame { get; set; }
-
-        /// <summary>
-        /// Keep machines that don't contain any items
-        /// </summary>
-        [JsonIgnore]
-        public bool KeepEmptyGames { get; set; }
-
-        /// <summary>
-        /// Remove scene dates from the beginning of machine names
-        /// </summary>
-        [JsonIgnore]
-        public bool SceneDateStrip { get; set; }
-
         /// <summary>
         /// Deduplicate items using the given method
         /// </summary>
@@ -315,7 +279,6 @@ namespace SabreTools.Library.DatFiles
         [JsonIgnore]
         public Hash StripHash { get; private set; }
 
-
         #endregion
 
         #region Write pre-processing
@@ -538,12 +501,6 @@ namespace SabreTools.Library.DatFiles
                 ForceNodump = this.ForceNodump,
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
-                ExcludeFields = this.ExcludeFields,
-                OneGamePerRegion = this.OneGamePerRegion,
-                RegionList = this.RegionList,
-                OneRomPerGame = this.OneRomPerGame,
-                KeepEmptyGames = this.KeepEmptyGames,
-                SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
                 StripHash = this.StripHash,
 
@@ -585,12 +542,6 @@ namespace SabreTools.Library.DatFiles
                 ForceNodump = this.ForceNodump,
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
-                ExcludeFields = this.ExcludeFields,
-                OneGamePerRegion = this.OneGamePerRegion,
-                RegionList = this.RegionList,
-                OneRomPerGame = this.OneRomPerGame,
-                KeepEmptyGames = this.KeepEmptyGames,
-                SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
                 StripHash = this.StripHash,
             };
@@ -604,10 +555,6 @@ namespace SabreTools.Library.DatFiles
             return new DatHeader()
             {
                 DatFormat = this.DatFormat,
-                ExcludeFields = this.ExcludeFields,
-                OneRomPerGame = this.OneRomPerGame,
-                KeepEmptyGames = this.KeepEmptyGames,
-                SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
                 StripHash = this.StripHash,
 
@@ -684,12 +631,6 @@ namespace SabreTools.Library.DatFiles
             if (datHeader.DatFormat != 0x00)
                 DatFormat = datHeader.DatFormat;
 
-            if (datHeader.ExcludeFields != null)
-                ExcludeFields = datHeader.ExcludeFields;
-
-            OneRomPerGame = datHeader.OneRomPerGame;
-            KeepEmptyGames = datHeader.KeepEmptyGames;
-            SceneDateStrip = datHeader.SceneDateStrip;
             DedupeRoms = datHeader.DedupeRoms;
             //StripHash = datHeader.StripHash;
 

SabreTools.Library/DatFiles/ItemDictionary.cs

@@ -715,6 +715,28 @@ namespace SabreTools.Library.DatFiles
             ClearEmpty();
         }
 
+        /// <summary>
+        /// Remove any keys that have null or empty values
+        /// </summary>
+        public void ClearEmpty()
+        {
+            var keys = items.Keys.Where(k => k != null).ToList();
+            foreach (string key in keys)
+            {
+                // If the key doesn't exist, skip
+                if (!items.ContainsKey(key))
+                    continue;
+
+                // If the value is null, remove
+                else if (items[key] == null)
+                    items.Remove(key);
+
+                // If there are no non-blank items, remove
+                else if (items[key].Count(i => i != null && i.ItemType != ItemType.Blank) == 0)
+                    items.Remove(key);
+            }
+        }
+
         /// <summary>
         /// Remove all items marked for removal
         /// </summary>
@@ -829,22 +851,6 @@ namespace SabreTools.Library.DatFiles
             });
         }
 
-        /// <summary>
-        /// Remove any keys that have null or empty values
-        /// </summary>
-        private void ClearEmpty()
-        {
-            var keys = items.Keys.Where(k => k != null).ToList();
-            foreach (string key in keys)
-            {
-                if (!items.ContainsKey(key))
-                    continue;
-
-                if (items[key] == null || items[key].Count == 0)
-                    items.Remove(key);
-            }
-        }
-
         /// <summary>
         /// Get the highest-order Field value that represents the statistics
         /// </summary>

SabreTools.Library/Filtering/Cleaner.cs

@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+
+using SabreTools.Library.DatItems;
+
+namespace SabreTools.Library.Filtering
+{
+    /// <summary>
+    /// Represents the cleaning operations that need to be performed on a set of items, usually a DAT
+    /// </summary>
+    public class Cleaner
+    {
+        /// <summary>
+        /// Clean all names to WoD standards
+        /// </summary>
+        public bool Clean { get; set; }
+
+        /// <summary>
+        /// Set Machine Description from Machine Name
+        /// </summary>
+        public bool DescriptionAsName { get; set; }
+
+        /// <summary>
+        /// Dictionary of fields in machine and items to exclude from writing
+        /// </summary>
+        public List<Field> ExcludeFields { get; set; } = new List<Field>();
+
+        /// <summary>
+        /// Keep machines that don't contain any items
+        /// </summary>
+        public bool KeepEmptyGames { get; set; }
+
+        /// <summary>
+        /// Enable "One Rom, One Region (1G1R)" mode
+        /// </summary>
+        public bool OneGamePerRegion { get; set; }
+
+        /// <summary>
+        /// Ordered list of regions for "One Rom, One Region (1G1R)" mode
+        /// </summary>
+        public List<string> RegionList { get; set; }
+
+        /// <summary>
+        /// Ensure each rom is in their own game
+        /// </summary>
+        public bool OneRomPerGame { get; set; }
+
+        /// <summary>
+        /// Remove all unicode characters
+        /// </summary>
+        public bool RemoveUnicode { get; set; }
+
+        /// <summary>
+        /// Include root directory when determing trim sizes
+        /// </summary>
+        public string Root { get; set; }
+
+        /// <summary>
+        /// Remove scene dates from the beginning of machine names
+        /// </summary>
+        public bool SceneDateStrip { get; set; }
+
+        /// <summary>
+        /// Change all machine names to "!"
+        /// </summary>
+        public bool Single { get; set; }
+
+        /// <summary>
+        /// Trim total machine and item name to not exceed NTFS limits
+        /// </summary>
+        public bool Trim { get; set; }
+    }
+}

SabreTools.Library/Filtering/Filter.cs

@@ -352,17 +352,7 @@ namespace SabreTools.Library.Filtering
 
         #endregion // DatItem Filters
 
-        #region Manipulation Flags
+        #region Additional Flags
-
-        /// <summary>
-        /// Clean all names to WoD standards
-        /// </summary>
-        public bool Clean { get; set; }
-
-        /// <summary>
-        /// Set Machine Description from Machine Name
-        /// </summary>
-        public bool DescriptionAsName { get; set; }
 
         /// <summary>
         /// Include romof and cloneof when filtering machine names
@@ -374,26 +364,6 @@ namespace SabreTools.Library.Filtering
         /// </summary>
         public MergingFlag InternalSplit { get; set; }
 
-        /// <summary>
-        /// Remove all unicode characters
-        /// </summary>
-        public bool RemoveUnicode { get; set; }
-
-        /// <summary>
-        /// Include root directory when determing trim sizes
-        /// </summary>
-        public string Root { get; set; }
-
-        /// <summary>
-        /// Change all machine names to "!"
-        /// </summary>
-        public bool Single { get; set; }
-
-        /// <summary>
-        /// Trim total machine and item name to not exceed NTFS limits
-        /// </summary>
-        public bool Trim { get; set; }
-
         #endregion
 
         #endregion // Fields

SabreTools/Features/BaseFeature.cs

@@ -2302,6 +2302,11 @@ Some special strings that can be used:
 
         #region Fields
 
+        /// <summary>
+        /// Preconfigured Cleaner
+        /// </summary>
+        protected Cleaner Cleaner { get; set; }
+
         /// <summary>
         /// Preconfigured ExtraIni set
         /// </summary>
@@ -2419,6 +2424,7 @@ Some special strings that can be used:
         public override void ProcessFeatures(Dictionary<string, Feature> features)
         {
             // Generic feature flags
+            Cleaner = GetCleaner(features);
             Extras = GetExtras(features);
             Filter = GetFilter(features);
             Header = GetDatHeader(features);
@@ -2665,6 +2671,34 @@ Some special strings that can be used:
 
         #region Private Specific Extraction
 
+        /// <summary>
+        /// Get Cleaner from feature list
+        /// </summary>
+        private Cleaner GetCleaner(Dictionary<string, Feature> features)
+        {
+            Cleaner cleaner = new Cleaner()
+            {
+                Clean = GetBoolean(features, CleanValue),
+                DescriptionAsName = GetBoolean(features, DescriptionAsNameValue),
+                KeepEmptyGames = GetBoolean(features, KeepEmptyGamesValue),
+                OneGamePerRegion = GetBoolean(features, OneGamePerRegionValue),
+                RegionList = GetList(features, RegionListValue),
+                OneRomPerGame = GetBoolean(features, OneRomPerGameValue),
+                RemoveUnicode = GetBoolean(features, RemoveUnicodeValue),
+                Root = GetString(features, RootDirStringValue),
+                SceneDateStrip = GetBoolean(features, SceneDateStripValue),
+                Single = GetBoolean(features, SingleSetValue),
+                Trim = GetBoolean(features, TrimValue),
+            };
+
+            foreach (string fieldName in GetList(features, ExcludeFieldListValue))
+            {
+                cleaner.ExcludeFields.Add(fieldName.AsField());
+            }
+
+            return cleaner;
+        }
+
         /// <summary>
         /// Get DatHeader from feature list
         /// </summary>
@@ -2688,18 +2722,13 @@ Some special strings that can be used:
                 GameName = GetBoolean(features, GamePrefixValue),
                 HeaderSkipper = GetString(features, HeaderStringValue),
                 Homepage = GetString(features, HomepageStringValue),
-                KeepEmptyGames = GetBoolean(features, KeepEmptyGamesValue),
                 Name = GetString(features, NameStringValue),
-                OneGamePerRegion = GetBoolean(features, OneGamePerRegionValue),
-                OneRomPerGame = GetBoolean(features, OneRomPerGameValue),
                 Postfix = GetString(features, PostfixStringValue),
                 Prefix = GetString(features, PrefixStringValue),
                 Quotes = GetBoolean(features, QuotesValue),
-                RegionList = GetList(features, RegionListValue),
                 RemoveExtension = GetBoolean(features, RemoveExtensionsValue),
                 ReplaceExtension = GetString(features, ReplaceExtensionStringValue),
                 RootDir = GetString(features, RootStringValue),
-                SceneDateStrip = GetBoolean(features, SceneDateStripValue),
                 Type = GetBoolean(features, SuperdatValue) ? "SuperDAT" : null,
                 Url = GetString(features, UrlStringValue),
                 UseRomName = GetBoolean(features, RomsValue),
@@ -2724,11 +2753,6 @@ Some special strings that can be used:
                     datHeader.DatFormat |= dftemp;
             }
 
-            foreach (string fieldName in GetList(features, ExcludeFieldListValue))
-            {
-                datHeader.ExcludeFields.Add(fieldName.AsField());
-            }
-
             return datHeader;
         }
 
@@ -2972,13 +2996,7 @@ Some special strings that can be used:
 
             #endregion
 
-            #region Filter manipulation flags
+            #region Additional Filter flags
-
-            // Clean names
-            filter.Clean = GetBoolean(features, CleanValue);
-
-            // Machine description as machine name
-            filter.DescriptionAsName = GetBoolean(features, DescriptionAsNameValue);
 
             // Include 'of" in game filters
             filter.IncludeOfInGame = GetBoolean(features, MatchOfTagsValue);
@@ -2986,18 +3004,6 @@ Some special strings that can be used:
             // Internal splitting
             filter.InternalSplit = GetSplitType(features);
 
-            // Remove unicode characters
-            filter.RemoveUnicode = GetBoolean(features, RemoveUnicodeValue);
-
-            // Root directory
-            filter.Root = GetString(features, RootDirStringValue);
-
-            // Single game in output
-            filter.Single = GetBoolean(features, SingleSetValue);
-
-            // Trim to NTFS length
-            filter.Trim = GetBoolean(features, TrimValue);
-
             #endregion
 
             return filter;

SabreTools/Features/Batch.cs

@@ -172,6 +172,7 @@ Reset the internal state:           reset();";
 
                                 // Apply the filter blindly
                                 datFile.ApplyFilter(filter, false);
+                                datFile.Items.ClearMarked(); // TODO: We might not want to remove immediately
 
                                 break;
 
@@ -240,15 +241,8 @@ Reset the internal state:           reset();";
                                     continue;
                                 }
 
-                                // Set the region list
-                                var tempRegionList = datFile.Header.RegionList;
-                                datFile.Header.RegionList = command.Arguments;
-
                                 // Run the 1G1R functionality
-                                datFile.OneGamePerRegion();
+                                datFile.OneGamePerRegion(command.Arguments);
-
-                                // Reset the header value
-                                datFile.Header.RegionList = tempRegionList;
 
                                 break;
 
@@ -275,15 +269,8 @@ Reset the internal state:           reset();";
                                     continue;
                                 }
 
-                                // Set the field list
-                                var tempRemoveFields = datFile.Header.ExcludeFields;
-                                datFile.Header.ExcludeFields = command.Arguments.Select(s => s.AsField()).ToList();
-
                                 // Run the removal functionality
-                                datFile.RemoveFieldsFromItems();
+                                datFile.RemoveFieldsFromItems(command.Arguments.Select(s => s.AsField()).ToList());
-
-                                // Reset the header value
-                                datFile.Header.ExcludeFields = tempRemoveFields;
 
                                 break;
 

SabreTools/Features/DatFromDir.cs

@@ -95,6 +95,7 @@ namespace SabreTools.Features
                     {
                         datdata.ApplyExtras(Extras);
                         datdata.ApplyFilter(Filter, false);
+                        datdata.ApplyCleaning(Cleaner);
                         datdata.Write(OutputDir);
                     }
                     else

SabreTools/Features/Update.cs

@@ -167,7 +167,8 @@ namespace SabreTools.Features
                             || datFile.Header.DatFormat.HasFlag(DatFormat.CSV)
                             || datFile.Header.DatFormat.HasFlag(DatFormat.SSV));
                     datFile.ApplyExtras(Extras);
-                    datFile.ApplyFilter(Filter, false /* useTags */);
+                    datFile.ApplyFilter(Filter, false);
+                    datFile.ApplyCleaning(Cleaner);
 
                     // Get the correct output path
                     string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
@@ -201,9 +202,10 @@ namespace SabreTools.Features
             else
                 datHeaders = userInputDat.PopulateUserData(inputPaths);
 
-            // Apply the extras and filter
+            // Apply the extras, filter, and cleaning
             userInputDat.ApplyExtras(Extras);
             userInputDat.ApplyFilter(Filter, false);
+            userInputDat.ApplyCleaning(Cleaner);
 
             // Output only DatItems that are duplicated across inputs
             if (updateMode.HasFlag(UpdateMode.DiffDupesOnly))
@@ -287,11 +289,12 @@ namespace SabreTools.Features
                 // Loop through each input and diff against the base
                 Parallel.ForEach(inputPaths, Globals.ParallelOptions, inputPath =>
                 {
-                    // Parse, extras, and filter the path to a new DatFile
+                    // Parse and process the path to a new DatFile
                     DatFile repDat = DatFile.Create(userInputDat.Header.CloneFiltering());
                     repDat.Parse(inputPath, indexId: 1, keep: true);
                     repDat.ApplyExtras(Extras);
                     repDat.ApplyFilter(Filter, false);
+                    repDat.ApplyCleaning(Cleaner);
 
                     // Now replace the fields from the base DatFile
                     userInputDat.DiffAgainst(repDat, GetBoolean(Features, ByGameValue));
@@ -308,11 +311,12 @@ namespace SabreTools.Features
                 // Loop through each input and apply the base DatFile
                 Parallel.ForEach(inputPaths, Globals.ParallelOptions, inputPath =>
                 {
-                    // Parse, extras, and filter the path to a new DatFile
+                    // Parse and process the path to a new DatFile
                     DatFile repDat = DatFile.Create(userInputDat.Header.CloneFiltering());
                     repDat.Parse(inputPath, indexId: 1, keep: true);
                     repDat.ApplyExtras(Extras);
                     repDat.ApplyFilter(Filter, false);
+                    repDat.ApplyCleaning(Cleaner);
 
                     // Now replace the fields from the base DatFile
                     userInputDat.BaseReplace(repDat, updateFields, GetBoolean(features, OnlySameValue));

SabreTools/Features/Verify.cs

@@ -58,6 +58,7 @@ namespace SabreTools.Features
                     datdata.Parse(datfile, 99, keep: true);
                     datdata.ApplyExtras(Extras);
                     datdata.ApplyFilter(Filter, true);
+                    datdata.ApplyCleaning(Cleaner);
 
                     // Set depot information
                     datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
@@ -88,6 +89,7 @@ namespace SabreTools.Features
                     datdata.Parse(datfile, 99, keep: true);
                     datdata.ApplyExtras(Extras);
                     datdata.ApplyFilter(Filter, true);
+                    datdata.ApplyCleaning(Cleaner);
                 }
 
                 // Set depot information