Move dedupe to Cleaner, remove StripHash

SabreTools.Library/DatFiles/DatFile.cs

@@ -709,6 +709,12 @@ namespace SabreTools.Library.DatFiles
                 // Perform item-level cleaning
                 CleanDatItems(cleaner);
 
+                // Bucket and dedupe according to the flag
+                if (cleaner?.DedupeRoms == DedupeType.Full)
+                    Items.BucketBy(Field.DatItem_CRC, cleaner.DedupeRoms);
+                else if (cleaner?.DedupeRoms == DedupeType.Game)
+                    Items.BucketBy(Field.Machine_Name, cleaner.DedupeRoms);
+
                 // Process description to machine name
                 if (cleaner?.DescriptionAsName == true)
                     MachineDescriptionToName();
@@ -3437,14 +3443,6 @@ namespace SabreTools.Library.DatFiles
                 consoleOutput.ReplaceStatistics(Header.FileName, Items.Keys.Count(), Items);
             }
 
-            // Bucket and dedupe according to the flag
-            // TODO: Can this be made into a cleaning flag instead of a header one?
-            // TODO: Should this be run as a cleaning step?
-            if (Header.DedupeRoms == DedupeType.Full)
-                Items.BucketBy(Field.DatItem_CRC, Header.DedupeRoms, norename: norename);
-            else if (Header.DedupeRoms == DedupeType.Game)
-                Items.BucketBy(Field.Machine_Name, Header.DedupeRoms, norename: norename);
-
             // Bucket roms by game name, if not already
             Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: norename);
 

SabreTools.Library/DatFiles/DatHeader.cs

@@ -265,22 +265,6 @@ namespace SabreTools.Library.DatFiles
 
         #endregion
 
-        #region Filtering Fields
-
-        /// <summary>
-        /// Deduplicate items using the given method
-        /// </summary>
-        [JsonIgnore]
-        public DedupeType DedupeRoms { get; set; }
-
-        /// <summary>
-        /// Strip hash types from items
-        /// </summary>
-        [JsonIgnore]
-        public Hash StripHash { get; private set; }
-
-        #endregion
-
         #region Write pre-processing
 
         /// <summary>
@@ -501,8 +485,6 @@ namespace SabreTools.Library.DatFiles
                 ForceNodump = this.ForceNodump,
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
-                DedupeRoms = this.DedupeRoms,
-                StripHash = this.StripHash,
 
                 UseRomName = this.UseRomName,
                 Prefix = this.Prefix,
@@ -542,8 +524,6 @@ namespace SabreTools.Library.DatFiles
                 ForceNodump = this.ForceNodump,
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
-                DedupeRoms = this.DedupeRoms,
-                StripHash = this.StripHash,
             };
         }
 
@@ -555,8 +535,6 @@ namespace SabreTools.Library.DatFiles
             return new DatHeader()
             {
                 DatFormat = this.DatFormat,
-                DedupeRoms = this.DedupeRoms,
-                StripHash = this.StripHash,
 
                 UseRomName = this.UseRomName,
                 Prefix = this.Prefix,
@@ -631,9 +609,6 @@ namespace SabreTools.Library.DatFiles
             if (datHeader.DatFormat != 0x00)
                 DatFormat = datHeader.DatFormat;
 
-            DedupeRoms = datHeader.DedupeRoms;
-            //StripHash = datHeader.StripHash;
-
             if (!string.IsNullOrWhiteSpace(datHeader.Prefix))
                 Prefix = datHeader.Prefix;
 

SabreTools.Library/Filtering/Cleaner.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 
+using SabreTools.Library.DatFiles;
 using SabreTools.Library.DatItems;
 
 namespace SabreTools.Library.Filtering
@@ -14,6 +15,12 @@ namespace SabreTools.Library.Filtering
         /// </summary>
         public bool Clean { get; set; }
 
+        /// <summary>
+        /// Deduplicate items using the given method
+        /// </summary>
+        public DedupeType DedupeRoms { get; set; }
+
+
         /// <summary>
         /// Set Machine Description from Machine Name
         /// </summary>

SabreTools/Features/BaseFeature.cs

@@ -2699,6 +2699,7 @@ Some special strings that can be used:
             Cleaner cleaner = new Cleaner()
             {
                 Clean = GetBoolean(features, CleanValue),
+                DedupeRoms = GetDedupeType(features),
                 DescriptionAsName = GetBoolean(features, DescriptionAsNameValue),
                 KeepEmptyGames = GetBoolean(features, KeepEmptyGamesValue),
                 OneGamePerRegion = GetBoolean(features, OneGamePerRegionValue),
@@ -2732,7 +2733,6 @@ Some special strings that can be used:
                 Category = GetString(features, CategoryStringValue),
                 Comment = GetString(features, CommentStringValue),
                 Date = GetString(features, DateStringValue),
-                DedupeRoms = GetDedupeType(features),
                 Description = GetString(features, DescriptionStringValue),
                 Email = GetString(features, EmailStringValue),
                 FileName = GetString(features, FilenameStringValue),

SabreTools/Features/Update.cs

@@ -125,14 +125,14 @@ namespace SabreTools.Features
                 {
                     Header.Name = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
                         + (Header.Type == "SuperDAT" ? "-SuperDAT" : string.Empty)
-                        + (Header.DedupeRoms != DedupeType.None ? "-deduped" : string.Empty);
+                        + (Cleaner.DedupeRoms != DedupeType.None ? "-deduped" : string.Empty);
                 }
 
                 if (string.IsNullOrWhiteSpace(Header.Description))
                 {
                     Header.Description = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
                         + (Header.Type == "SuperDAT" ? "-SuperDAT" : string.Empty)
-                        + (Header.DedupeRoms != DedupeType.None ? " - deduped" : string.Empty);
+                        + (Cleaner.DedupeRoms != DedupeType.None ? " - deduped" : string.Empty);
 
                     if (!GetBoolean(features, NoAutomaticDateValue))
                         Header.Description += $" ({Header.Date})";