From 971dbfadff729a2ac6e71be1d223236f1d4287c6 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 12 Feb 2025 14:22:15 -0500 Subject: [PATCH] Consolidate replacement methods --- SabreTools.DatFiles/DatFileTool.cs | 222 ---------------------------- SabreTools.DatFiles/Replacer.cs | 223 +++++++++++++++++++++++++++++ SabreTools/Features/Update.cs | 4 +- 3 files changed, 225 insertions(+), 224 deletions(-) diff --git a/SabreTools.DatFiles/DatFileTool.cs b/SabreTools.DatFiles/DatFileTool.cs index f5299c31..c7cde689 100644 --- a/SabreTools.DatFiles/DatFileTool.cs +++ b/SabreTools.DatFiles/DatFileTool.cs @@ -440,228 +440,6 @@ namespace SabreTools.DatFiles #endregion - #region Replacement - - /// - /// Replace item values from the base set represented by the current DAT - /// - /// Current DatFile object to use for updating - /// DatFile to replace the values in - /// List of machine field names representing what should be updated - /// List of item field names representing what should be updated - /// True if descriptions should only be replaced if the game name is the same, false otherwise - public static void BaseReplace( - DatFile datFile, - DatFile intDat, - List machineFieldNames, - Dictionary> itemFieldNames, - bool onlySame) - { - InternalStopwatch watch = new($"Replacing items in '{intDat.Header.GetStringFieldValue(DatHeader.FileNameKey)}' from the base DAT"); - - // If we are matching based on DatItem fields of any sort - if (itemFieldNames.Count > 0) - { - // For comparison's sake, we want to use CRC as the base bucketing - datFile.BucketBy(ItemKey.CRC); - datFile.Deduplicate(); - intDat.BucketBy(ItemKey.CRC); - - // Then we do a hashwise comparison against the base DAT -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(intDat.Items.SortedKeys, Core.Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(intDat.Items.SortedKeys, key => -#else - foreach (var key in intDat.Items.SortedKeys) -#endif - { - List? datItems = intDat.GetItemsForBucket(key); - if (datItems == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - List newDatItems = []; - foreach (DatItem datItem in datItems) - { - List dupes = datFile.GetDuplicates(datItem, sorted: true); - if (datItem.Clone() is not DatItem newDatItem) - continue; - - // Replace fields from the first duplicate, if we have one - if (dupes.Count > 0) - Replacer.ReplaceFields(newDatItem, dupes[0], itemFieldNames); - - newDatItems.Add(newDatItem); - } - - // Now add the new list to the key - intDat.RemoveBucket(key); - newDatItems.ForEach(item => intDat.AddItem(item, statsOnly: false)); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - - // If we are matching based on Machine fields of any sort - if (machineFieldNames.Count > 0) - { - // For comparison's sake, we want to use Machine Name as the base bucketing - datFile.BucketBy(ItemKey.Machine); - datFile.Deduplicate(); - intDat.BucketBy(ItemKey.Machine); - - // Then we do a namewise comparison against the base DAT -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(intDat.Items.SortedKeys, Core.Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(intDat.Items.SortedKeys, key => -#else - foreach (var key in intDat.Items.SortedKeys) -#endif - { - List? datItems = intDat.GetItemsForBucket(key); - if (datItems == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - List newDatItems = []; - foreach (DatItem datItem in datItems) - { - if (datItem.Clone() is not DatItem newDatItem) - continue; - - var list = datFile.GetItemsForBucket(key); - if (list.Count > 0) - Replacer.ReplaceFields(newDatItem.GetFieldValue(DatItem.MachineKey)!, list[index: 0].GetFieldValue(DatItem.MachineKey)!, machineFieldNames, onlySame); - - newDatItems.Add(newDatItem); - } - - // Now add the new list to the key - intDat.RemoveBucket(key); - newDatItems.ForEach(item => intDat.AddItem(item, statsOnly: false)); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - - watch.Stop(); - } - - /// - /// Replace item values from the base set represented by the current DAT - /// - /// Current DatFile object to use for updating - /// DatFile to replace the values in - /// List of machine field names representing what should be updated - /// List of item field names representing what should be updated - /// True if descriptions should only be replaced if the game name is the same, false otherwise - public static void BaseReplaceDB( - DatFile datFile, - DatFile intDat, - List machineFieldNames, - Dictionary> itemFieldNames, - bool onlySame) - { - InternalStopwatch watch = new($"Replacing items in '{intDat.Header.GetStringFieldValue(DatHeader.FileNameKey)}' from the base DAT"); - - // If we are matching based on DatItem fields of any sort - if (itemFieldNames.Count > 0) - { - // For comparison's sake, we want to use CRC as the base bucketing - datFile.BucketBy(ItemKey.CRC); - datFile.Deduplicate(); - intDat.BucketBy(ItemKey.CRC); - - // Then we do a hashwise comparison against the base DAT -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(intDat.ItemsDB.SortedKeys, Core.Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(intDat.ItemsDB.SortedKeys, key => -#else - foreach (var key in intDat.ItemsDB.SortedKeys) -#endif - { - var datItems = intDat.GetItemsForBucketDB(key); - if (datItems == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - foreach (var datItem in datItems) - { - var dupes = datFile.GetDuplicatesDB(datItem, sorted: true); - if (datItem.Value.Clone() is not DatItem newDatItem) - continue; - - // Replace fields from the first duplicate, if we have one - if (dupes.Count > 0) - Replacer.ReplaceFields(datItem.Value, dupes.First().Value, itemFieldNames); - } -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - - // If we are matching based on Machine fields of any sort - if (machineFieldNames.Count > 0) - { - // For comparison's sake, we want to use Machine Name as the base bucketing - datFile.BucketBy(ItemKey.Machine); - datFile.Deduplicate(); - intDat.BucketBy(ItemKey.Machine); - - // Then we do a namewise comparison against the base DAT -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(intDat.ItemsDB.SortedKeys, Core.Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(intDat.ItemsDB.SortedKeys, key => -#else - foreach (var key in intDat.ItemsDB.SortedKeys) -#endif - { - var datItems = intDat.GetItemsForBucketDB(key); - if (datItems == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - foreach (var datItem in datItems) - { - var datMachine = datFile.ItemsDB.GetMachineForItem(datFile.GetItemsForBucketDB(key)!.First().Key); - var intMachine = intDat.ItemsDB.GetMachineForItem(datItem.Key); - if (datMachine.Value != null && intMachine.Value != null) - Replacer.ReplaceFields(intMachine.Value, datMachine.Value, machineFieldNames, onlySame); - } -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - - watch.Stop(); - } - - #endregion - #region Population /// diff --git a/SabreTools.DatFiles/Replacer.cs b/SabreTools.DatFiles/Replacer.cs index 40e7a2dd..dcbab488 100644 --- a/SabreTools.DatFiles/Replacer.cs +++ b/SabreTools.DatFiles/Replacer.cs @@ -1,7 +1,12 @@ using System.Collections.Generic; +using System.Linq; +#if NET40_OR_GREATER || NETCOREAPP +using System.Threading.Tasks; +#endif using SabreTools.Core.Tools; using SabreTools.DatItems; using SabreTools.DatItems.Formats; +using SabreTools.IO.Logging; namespace SabreTools.DatFiles { @@ -10,6 +15,224 @@ namespace SabreTools.DatFiles /// public static class Replacer { + /// + /// Replace item values from the base set represented by the current DAT + /// + /// Current DatFile object to use for updating + /// DatFile to replace the values in + /// List of machine field names representing what should be updated + /// List of item field names representing what should be updated + /// True if descriptions should only be replaced if the game name is the same, false otherwise + public static void BaseReplace( + DatFile datFile, + DatFile intDat, + List machineFieldNames, + Dictionary> itemFieldNames, + bool onlySame) + { + InternalStopwatch watch = new($"Replacing items in '{intDat.Header.GetStringFieldValue(DatHeader.FileNameKey)}' from the base DAT"); + + // If we are matching based on DatItem fields of any sort + if (itemFieldNames.Count > 0) + { + // For comparison's sake, we want to use CRC as the base bucketing + datFile.BucketBy(ItemKey.CRC); + datFile.Deduplicate(); + intDat.BucketBy(ItemKey.CRC); + + // Then we do a hashwise comparison against the base DAT +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(intDat.Items.SortedKeys, Core.Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(intDat.Items.SortedKeys, key => +#else + foreach (var key in intDat.Items.SortedKeys) +#endif + { + List? datItems = intDat.GetItemsForBucket(key); + if (datItems == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + List newDatItems = []; + foreach (DatItem datItem in datItems) + { + List dupes = datFile.GetDuplicates(datItem, sorted: true); + if (datItem.Clone() is not DatItem newDatItem) + continue; + + // Replace fields from the first duplicate, if we have one + if (dupes.Count > 0) + Replacer.ReplaceFields(newDatItem, dupes[0], itemFieldNames); + + newDatItems.Add(newDatItem); + } + + // Now add the new list to the key + intDat.RemoveBucket(key); + newDatItems.ForEach(item => intDat.AddItem(item, statsOnly: false)); +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + + // If we are matching based on Machine fields of any sort + if (machineFieldNames.Count > 0) + { + // For comparison's sake, we want to use Machine Name as the base bucketing + datFile.BucketBy(ItemKey.Machine); + datFile.Deduplicate(); + intDat.BucketBy(ItemKey.Machine); + + // Then we do a namewise comparison against the base DAT +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(intDat.Items.SortedKeys, Core.Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(intDat.Items.SortedKeys, key => +#else + foreach (var key in intDat.Items.SortedKeys) +#endif + { + List? datItems = intDat.GetItemsForBucket(key); + if (datItems == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + List newDatItems = []; + foreach (DatItem datItem in datItems) + { + if (datItem.Clone() is not DatItem newDatItem) + continue; + + var list = datFile.GetItemsForBucket(key); + if (list.Count > 0) + Replacer.ReplaceFields(newDatItem.GetFieldValue(DatItem.MachineKey)!, list[index: 0].GetFieldValue(DatItem.MachineKey)!, machineFieldNames, onlySame); + + newDatItems.Add(newDatItem); + } + + // Now add the new list to the key + intDat.RemoveBucket(key); + newDatItems.ForEach(item => intDat.AddItem(item, statsOnly: false)); +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + + watch.Stop(); + } + + /// + /// Replace item values from the base set represented by the current DAT + /// + /// Current DatFile object to use for updating + /// DatFile to replace the values in + /// List of machine field names representing what should be updated + /// List of item field names representing what should be updated + /// True if descriptions should only be replaced if the game name is the same, false otherwise + public static void BaseReplaceDB( + DatFile datFile, + DatFile intDat, + List machineFieldNames, + Dictionary> itemFieldNames, + bool onlySame) + { + InternalStopwatch watch = new($"Replacing items in '{intDat.Header.GetStringFieldValue(DatHeader.FileNameKey)}' from the base DAT"); + + // If we are matching based on DatItem fields of any sort + if (itemFieldNames.Count > 0) + { + // For comparison's sake, we want to use CRC as the base bucketing + datFile.BucketBy(ItemKey.CRC); + datFile.Deduplicate(); + intDat.BucketBy(ItemKey.CRC); + + // Then we do a hashwise comparison against the base DAT +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(intDat.ItemsDB.SortedKeys, Core.Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(intDat.ItemsDB.SortedKeys, key => +#else + foreach (var key in intDat.ItemsDB.SortedKeys) +#endif + { + var datItems = intDat.GetItemsForBucketDB(key); + if (datItems == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + foreach (var datItem in datItems) + { + var dupes = datFile.GetDuplicatesDB(datItem, sorted: true); + if (datItem.Value.Clone() is not DatItem newDatItem) + continue; + + // Replace fields from the first duplicate, if we have one + if (dupes.Count > 0) + Replacer.ReplaceFields(datItem.Value, dupes.First().Value, itemFieldNames); + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + + // If we are matching based on Machine fields of any sort + if (machineFieldNames.Count > 0) + { + // For comparison's sake, we want to use Machine Name as the base bucketing + datFile.BucketBy(ItemKey.Machine); + datFile.Deduplicate(); + intDat.BucketBy(ItemKey.Machine); + + // Then we do a namewise comparison against the base DAT +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(intDat.ItemsDB.SortedKeys, Core.Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(intDat.ItemsDB.SortedKeys, key => +#else + foreach (var key in intDat.ItemsDB.SortedKeys) +#endif + { + var datItems = intDat.GetItemsForBucketDB(key); + if (datItems == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + foreach (var datItem in datItems) + { + var datMachine = datFile.ItemsDB.GetMachineForItem(datFile.GetItemsForBucketDB(key)!.First().Key); + var intMachine = intDat.ItemsDB.GetMachineForItem(datItem.Key); + if (datMachine.Value != null && intMachine.Value != null) + Replacer.ReplaceFields(intMachine.Value, datMachine.Value, machineFieldNames, onlySame); + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + + watch.Stop(); + } + /// /// Replace fields with given values /// diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index a1d15639..99499370 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -444,8 +444,8 @@ namespace SabreTools.Features Remover.ApplyRemovals(repDat); // Now replace the fields from the base DatFile - DatFileTool.BaseReplace( - //DatFileTool.BaseReplaceDB( + Replacer.BaseReplace( + //Replacer.BaseReplaceDB( userInputDat, repDat, updateMachineFieldNames,