diff --git a/SabreTools.Core/Enums.cs b/SabreTools.Core/Enums.cs index 5fb75c31..00b10ae6 100644 --- a/SabreTools.Core/Enums.cs +++ b/SabreTools.Core/Enums.cs @@ -23,7 +23,7 @@ namespace SabreTools.Core Standard = CRC | MD5 | SHA1, DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum, SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, - All = CRC | MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, + All = CRC | MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, } /// @@ -35,10 +35,15 @@ namespace SabreTools.Core Split, Merged, NonMerged, - Full, /// This is not usually defined for Merging flags - Device, + FullMerged, + + /// This is not usually defined for Merging flags + DeviceNonMerged, + + /// This is not usually defined for Merging flags + FullNonMerged, } /// diff --git a/SabreTools.Core/README.1ST b/SabreTools.Core/README.1ST index 8fc942c4..085e94e1 100644 --- a/SabreTools.Core/README.1ST +++ b/SabreTools.Core/README.1ST @@ -608,12 +608,10 @@ Features and Options: children based on the cloneof tag. This is incompatible with the other --dat-X flags. - --frp, --force-rom-parenting Force ROMs to be added to parent - By default, a merged DAT will take the first instance of a given - ROM in the parent as the file existing. To be more strict to the - source, this flag allows overriding that where if the file is in - the child, it will be added to the resulting combined parent in - call cases that are not controlled by a merge tag. + -dfm, --dat-full-merged Force creating fully merged sets + Preprocess the DAT to have parent sets contain all items from the + children based on the cloneof tag while also performing deduplication + within a parent. This is incompatible with the other --dat-X flags. -ds, --dat-split Force creating split sets Preprocess the DAT to remove redundant files between parents and @@ -1010,12 +1008,10 @@ Features and Options: children based on the cloneof tag. This is incompatible with the other --dat-X flags. - --frp, --force-rom-parenting Force ROMs to be added to parent - By default, a merged DAT will take the first instance of a given - ROM in the parent as the file existing. To be more strict to the - source, this flag allows overriding that where if the file is in - the child, it will be added to the resulting combined parent in - call cases that are not controlled by a merge tag. + -dfm, --dat-full-merged Force creating fully merged sets + Preprocess the DAT to have parent sets contain all items from the + children based on the cloneof tag while also performing deduplication + within a parent. This is incompatible with the other --dat-X flags. -ds, --dat-split Force creating split sets Preprocess the DAT to remove redundant files between parents and @@ -1264,12 +1260,10 @@ Features and Options: children based on the cloneof tag. This is incompatible with the other --dat-X flags. - --frp, --force-rom-parenting Force ROMs to be added to parent - By default, a merged DAT will take the first instance of a given - ROM in the parent as the file existing. To be more strict to the - source, this flag allows overriding that where if the file is in - the child, it will be added to the resulting combined parent in - call cases that are not controlled by a merge tag. + -dfm, --dat-full-merged Force creating fully merged sets + Preprocess the DAT to have parent sets contain all items from the + children based on the cloneof tag while also performing deduplication + within a parent. This is incompatible with the other --dat-X flags. -ds, --dat-split Force creating split sets Preprocess the DAT to remove redundant files between parents and diff --git a/SabreTools.Core/Tools/Converters.cs b/SabreTools.Core/Tools/Converters.cs index ec4a7dcc..82b03127 100644 --- a/SabreTools.Core/Tools/Converters.cs +++ b/SabreTools.Core/Tools/Converters.cs @@ -1267,9 +1267,15 @@ namespace SabreTools.Core.Tools { "split" => MergingFlag.Split, "merged" => MergingFlag.Merged, + "fullmerged" => MergingFlag.FullMerged, "nonmerged" => MergingFlag.NonMerged, "unmerged" => MergingFlag.NonMerged, - "full" => MergingFlag.Full, + "full" => MergingFlag.FullNonMerged, + "fullnonmerged" => MergingFlag.FullNonMerged, + "fullunmerged" => MergingFlag.FullNonMerged, + "device" => MergingFlag.DeviceNonMerged, + "devicenonmerged" => MergingFlag.DeviceNonMerged, + "deviceunmerged" => MergingFlag.DeviceNonMerged, "none" => MergingFlag.None, _ => MergingFlag.None, }; @@ -1710,9 +1716,10 @@ namespace SabreTools.Core.Tools { MergingFlag.Split => "split", MergingFlag.Merged => "merged", + MergingFlag.FullMerged => "fullmerged", MergingFlag.NonMerged => romCenter ? "unmerged" : "nonmerged", - MergingFlag.Full => "full", - MergingFlag.Device => "device", + MergingFlag.FullNonMerged => "full", + MergingFlag.DeviceNonMerged => "device", _ => null, }; } diff --git a/SabreTools.DatFiles/Formats/RomCenter.cs b/SabreTools.DatFiles/Formats/RomCenter.cs index a6dfc8cf..fc2d055d 100644 --- a/SabreTools.DatFiles/Formats/RomCenter.cs +++ b/SabreTools.DatFiles/Formats/RomCenter.cs @@ -450,7 +450,7 @@ namespace SabreTools.DatFiles.Formats iw.WriteKeyValuePair("version", Header.RomCenterVersion ?? "2.50"); iw.WriteKeyValuePair("plugin", Header.System); iw.WriteKeyValuePair("split", Header.ForceMerging == MergingFlag.Split ? "1" : "0"); - iw.WriteKeyValuePair("merge", Header.ForceMerging == MergingFlag.Full || Header.ForceMerging == MergingFlag.Merged ? "1" : "0"); + iw.WriteKeyValuePair("merge", Header.ForceMerging == MergingFlag.FullNonMerged || Header.ForceMerging == MergingFlag.Merged ? "1" : "0"); iw.WriteSection("EMULATOR"); iw.WriteKeyValuePair("refname", Header.Name); diff --git a/SabreTools.Filtering/Splitter.cs b/SabreTools.Filtering/Splitter.cs index fda1c8bd..bbe0ec85 100644 --- a/SabreTools.Filtering/Splitter.cs +++ b/SabreTools.Filtering/Splitter.cs @@ -29,7 +29,7 @@ namespace SabreTools.Filtering private static readonly Logger logger = new Logger(); #endregion - + // TODO: Should any of these create a new DatFile in the process? // The reason this comes up is that doing any of the splits or merges // is an inherently destructive process. Making it output a new DatFile @@ -43,10 +43,9 @@ namespace SabreTools.Filtering /// /// Current DatFile object to run operations on /// True if DatFile tags override splitting, false otherwise - /// True to force adding ROMs to parent, false otherwise /// True if the error that is thrown should be thrown back to the caller, false otherwise /// True if the DatFile was split, false on error - public bool ApplySplitting(DatFile datFile, bool useTags, bool forceAddRoms, bool throwOnError = false) + public bool ApplySplitting(DatFile datFile, bool useTags, bool throwOnError = false) { InternalStopwatch watch = new InternalStopwatch("Applying splitting to DAT"); @@ -59,23 +58,29 @@ namespace SabreTools.Filtering // Run internal splitting switch (SplitType) { + // Standard case MergingFlag.None: // No-op break; - case MergingFlag.Device: - CreateDeviceNonMergedSets(datFile); + case MergingFlag.Split: + CreateSplitSets(datFile); break; - case MergingFlag.Full: - CreateFullyNonMergedSets(datFile); + case MergingFlag.Merged: + CreateMergedSets(datFile); break; case MergingFlag.NonMerged: CreateNonMergedSets(datFile); break; - case MergingFlag.Merged: - CreateMergedSets(datFile, forceAddRoms); + + // Nonstandard + case MergingFlag.FullMerged: + CreateFullyMergedSets(datFile); break; - case MergingFlag.Split: - CreateSplitSets(datFile); + case MergingFlag.DeviceNonMerged: + CreateDeviceNonMergedSets(datFile); + break; + case MergingFlag.FullNonMerged: + CreateFullyNonMergedSets(datFile); break; } } @@ -111,6 +116,28 @@ namespace SabreTools.Filtering RemoveTagsFromChild(datFile); } + /// + /// Use cloneof tags to create merged sets and remove the tags plus deduplicating if tags don't catch everything + /// + /// Current DatFile object to run operations on + internal static void CreateFullyMergedSets(DatFile datFile) + { + logger.User("Creating fully merged sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); + + // Now we want to loop through all of the games and set the correct information + AddRomsFromChildren(datFile, skipDedup: false); + + // Now that we have looped through the cloneof tags, we loop through the romof tags + RemoveBiosRomsFromChild(datFile, false); + RemoveBiosRomsFromChild(datFile, true); + + // Finally, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(datFile); + } + /// /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets /// @@ -138,8 +165,7 @@ namespace SabreTools.Filtering /// Use cloneof tags to create merged sets and remove the tags /// /// Current DatFile object to run operations on - /// True to force adding ROMs to parent, false otherwise - internal static void CreateMergedSets(DatFile datFile, bool forceAddRoms = false) + internal static void CreateMergedSets(DatFile datFile) { logger.User("Creating merged sets from the DAT"); @@ -147,7 +173,7 @@ namespace SabreTools.Filtering datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Now we want to loop through all of the games and set the correct information - AddRomsFromChildren(datFile, forceAddRoms: forceAddRoms); + AddRomsFromChildren(datFile, skipDedup: true); // Now that we have looped through the cloneof tags, we loop through the romof tags RemoveBiosRomsFromChild(datFile, false); @@ -425,8 +451,8 @@ namespace SabreTools.Filtering /// /// Current DatFile object to run operations on /// True to add DatItems to subfolder of parent (not including Disk), false otherwise - /// True to force adding ROMs to parent, false otherwise (requires subfolder) - internal static void AddRomsFromChildren(DatFile datFile, bool subfolder = true, bool forceAddRoms = false) + /// True to skip checking for duplicate ROMs in parent, false otherwise + internal static void AddRomsFromChildren(DatFile datFile, bool subfolder = true, bool skipDedup = false) { List games = datFile.Items.Keys.OrderBy(g => g).ToList(); foreach (string game in games) @@ -508,7 +534,7 @@ namespace SabreTools.Filtering } // If the parent doesn't already contain this item, add to subfolder of parent - else if (!datFile.Items[parent].Contains(item) || (subfolder && forceAddRoms)) + else if (!datFile.Items[parent].Contains(item) || skipDedup) { if (subfolder) rom.Name = $"{item.Machine.Name}\\{rom.Name}"; diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index a92f5a42..7a4177e0 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -192,6 +192,20 @@ namespace SabreTools.Features } } + internal const string DatFullMergedValue = "dat-full-merged"; + internal static Feature DatFullMergedFlag + { + get + { + return new Feature( + DatFullMergedValue, + new List() { "-dfm", "--dat-full-merged" }, + "Create fully merged sets", + ParameterType.Flag, + longDescription: "Preprocess the DAT to have parent sets contain all items from the children based on the cloneof tag while also performing deduplication within a parent. This is incompatible with the other --dat-X flags."); + } + } + internal const string DatFullNonMergedValue = "dat-full-non-merged"; internal static Feature DatFullNonMergedFlag { @@ -430,21 +444,6 @@ namespace SabreTools.Features } } - internal const string ForceRomParentingValue = "force-rom-parenting"; - internal static Feature ForceRomParentingFlag - { - get - { - return new Feature( - ForceRomParentingValue, - new List() { "-frp", "--force-rom-parenting" }, - "Force ROMs to be added to parent", - ParameterType.Flag, - "By default, a merged DAT will take the first instance of a given ROM in the parent as the file existing. To be more strict to the source, this flag allows overriding that where if the file is in the child, it will be added to the resulting combined parent in call cases that are not controlled by a merge tag." - ); - } - } - internal const string GameDedupValue = "game-dedup"; internal static Feature GameDedupFlag { @@ -1879,7 +1878,7 @@ Some special strings that can be used: protected void AddInternalSplitFeatures() { AddFeature(DatMergedFlag); - this[DatMergedFlag].AddFeature(ForceRomParentingFlag); + AddFeature(DatFullMergedFlag); AddFeature(DatSplitFlag); AddFeature(DatNonMergedFlag); AddFeature(DatDeviceNonMergedFlag); @@ -2277,9 +2276,11 @@ Some special strings that can be used: { MergingFlag splitType = MergingFlag.None; if (GetBoolean(features, DatDeviceNonMergedValue)) - splitType = MergingFlag.Device; + splitType = MergingFlag.DeviceNonMerged; + else if (GetBoolean(features, DatFullMergedValue)) + splitType = MergingFlag.FullMerged; else if (GetBoolean(features, DatFullNonMergedValue)) - splitType = MergingFlag.Full; + splitType = MergingFlag.FullNonMerged; else if (GetBoolean(features, DatMergedValue)) splitType = MergingFlag.Merged; else if (GetBoolean(features, DatNonMergedValue)) diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index 8382e7b4..8e2b3889 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -63,7 +63,6 @@ namespace SabreTools.Features bool addBlankFiles = GetBoolean(features, AddBlankFilesValue); bool addFileDates = GetBoolean(features, AddDateValue); TreatAsFile asFiles = GetTreatAsFiles(features); - bool forceAddRoms = GetBoolean(features, ForceRomParentingValue); bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue); var includeInScan = GetIncludeInScan(features); var skipFileType = GetSkipFileType(features); @@ -101,7 +100,7 @@ namespace SabreTools.Features { // Perform additional processing steps Extras.ApplyExtras(datdata); - Splitter.ApplySplitting(datdata, useTags: false, forceAddRoms); + Splitter.ApplySplitting(datdata, useTags: false); Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); Remover.ApplyRemovals(datdata); diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index d558a24a..bd36c839 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -91,7 +91,6 @@ namespace SabreTools.Features return false; // Get feature flags - bool forceAddRoms = GetBoolean(features, ForceRomParentingValue); var updateDatItemFields = GetUpdateDatItemFields(features); var updateMachineFields = GetUpdateMachineFields(features); var updateMode = GetUpdateMode(features); @@ -166,7 +165,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datFile); - Splitter.ApplySplitting(datFile, useTags: false, forceAddRoms); + Splitter.ApplySplitting(datFile, useTags: false); Filter.ApplyFilters(datFile); Cleaner.ApplyCleaning(datFile); Remover.ApplyRemovals(datFile); @@ -205,7 +204,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(userInputDat); - Splitter.ApplySplitting(userInputDat, useTags: false, forceAddRoms); + Splitter.ApplySplitting(userInputDat, useTags: false); Filter.ApplyFilters(userInputDat); Cleaner.ApplyCleaning(userInputDat); Remover.ApplyRemovals(userInputDat); @@ -298,7 +297,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); - Splitter.ApplySplitting(repDat, useTags: false, forceAddRoms); + Splitter.ApplySplitting(repDat, useTags: false); Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); Remover.ApplyRemovals(repDat); @@ -324,7 +323,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(repDat); - Splitter.ApplySplitting(repDat, useTags: false, forceAddRoms); + Splitter.ApplySplitting(repDat, useTags: false); Filter.ApplyFilters(repDat); Cleaner.ApplyCleaning(repDat); Remover.ApplyRemovals(repDat); diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 42a2d7d4..9634fdf6 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -53,7 +53,6 @@ namespace SabreTools.Features // Get feature flags TreatAsFile asFiles = GetTreatAsFiles(features); - bool forceAddRoms = GetBoolean(features, ForceRomParentingValue); bool hashOnly = GetBoolean(features, HashOnlyValue); bool quickScan = GetBoolean(features, QuickValue); @@ -68,7 +67,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); - Splitter.ApplySplitting(datdata, useTags: true, forceAddRoms); + Splitter.ApplySplitting(datdata, useTags: true); Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); Remover.ApplyRemovals(datdata); @@ -116,7 +115,7 @@ namespace SabreTools.Features // Perform additional processing steps Extras.ApplyExtras(datdata); - Splitter.ApplySplitting(datdata, useTags: true, forceAddRoms); + Splitter.ApplySplitting(datdata, useTags: true); Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); Remover.ApplyRemovals(datdata);