diff --git a/SabreTools.DatFiles/ItemDictionary.cs b/SabreTools.DatFiles/ItemDictionary.cs index 957470c3..c2b97605 100644 --- a/SabreTools.DatFiles/ItemDictionary.cs +++ b/SabreTools.DatFiles/ItemDictionary.cs @@ -1,4 +1,5 @@ -using System.Collections; +using System; +using System.Collections; #if NET40_OR_GREATER || NETCOREAPP using System.Collections.Concurrent; #endif @@ -853,6 +854,106 @@ namespace SabreTools.DatFiles #endif } + /// + /// Use game descriptions as names, updating cloneof/romof/sampleof + /// + /// True if the error that is thrown should be thrown back to the caller, false otherwise + public void MachineDescriptionToName(bool throwOnError = false) + { + try + { + // First we want to get a mapping for all games to description +#if NET40_OR_GREATER || NETCOREAPP + ConcurrentDictionary mapping = new(); +#else + Dictionary mapping = []; +#endif +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(Keys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(Keys, key => +#else + foreach (var key in Keys) +#endif + { + var items = this[key]; + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + foreach (DatItem item in items) + { + // If the key mapping doesn't exist, add it +#if NET40_OR_GREATER || NETCOREAPP + mapping.TryAdd(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!, item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); +#else + mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!] = item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -"); +#endif + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + // Now we loop through every item and update accordingly +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(Keys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(Keys, key => +#else + foreach (var key in Keys) +#endif + { + var items = this[key]; + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + ConcurrentList newItems = []; + foreach (DatItem item in items) + { + // Update machine name + if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!)) + item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.NameKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!]); + + // Update cloneof + if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!)) + item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.CloneOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!]); + + // Update romof + if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!)) + item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.RomOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!]); + + // Update sampleof + if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!)) + item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.SampleOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!]); + + // Add the new item to the output list + newItems.Add(item); + } + + // Replace the old list of roms with the new one + Remove(key); + AddRange(key, newItems); +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + catch (Exception ex) when (!throwOnError) + { + logger.Warning(ex.ToString()); + } + } + #endregion #region Statistics diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs index 12ef53c1..8a6e1ff4 100644 --- a/SabreTools.DatFiles/ItemDictionaryDB.cs +++ b/SabreTools.DatFiles/ItemDictionaryDB.cs @@ -1,4 +1,5 @@ -#if NET40_OR_GREATER || NETCOREAPP +using System; +#if NET40_OR_GREATER || NETCOREAPP using System.Collections.Concurrent; #endif using System.Collections.Generic; @@ -15,6 +16,7 @@ using SabreTools.Core.Tools; using SabreTools.DatItems; using SabreTools.DatItems.Formats; using SabreTools.Hashing; +using SabreTools.Logging; using SabreTools.Matching; /* @@ -101,6 +103,11 @@ namespace SabreTools.DatFiles /// private ItemKey _bucketedBy = ItemKey.NULL; + /// + /// Logging object + /// + private readonly Logger logger; + #endregion #region Fields @@ -128,6 +135,14 @@ namespace SabreTools.DatFiles #endregion + /// + /// Generic constructor + /// + public ItemDictionaryDB() + { + logger = new Logger(this); + } + #region Accessors /// @@ -901,6 +916,107 @@ namespace SabreTools.DatFiles #endif } + /// + /// Use game descriptions as names, updating cloneof/romof/sampleof + /// + /// True if the error that is thrown should be thrown back to the caller, false otherwise + public void MachineDescriptionToName(bool throwOnError = false) + { + try + { + // First we want to get a mapping for all games to description +#if NET40_OR_GREATER || NETCOREAPP + ConcurrentDictionary mapping = new(); +#else + Dictionary mapping = []; +#endif +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(SortedKeys, key => +#else + foreach (var key in SortedKeys) +#endif + { + var items = GetDatItemsForBucket(key); + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + foreach ((long, DatItem) item in items) + { + // If the key mapping doesn't exist, add it +#if NET40_OR_GREATER || NETCOREAPP + mapping.TryAdd(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!, + item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); +#else + mapping[item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!] + = item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -"); +#endif + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + // Now we loop through every item and update accordingly +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(SortedKeys, key => +#else + foreach (var key in SortedKeys) +#endif + { + var items = GetDatItemsForBucket(key); + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + ConcurrentList<(long, DatItem)> newItems = []; + foreach ((long, DatItem) item in items) + { + // Update machine name + if (!string.IsNullOrEmpty(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)) && mapping.ContainsKey(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!)) + item.Item2.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.NameKey, mapping[item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!]); + + // Update cloneof + if (!string.IsNullOrEmpty(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)) && mapping.ContainsKey(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!)) + item.Item2.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.CloneOfKey, mapping[item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!]); + + // Update romof + if (!string.IsNullOrEmpty(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)) && mapping.ContainsKey(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!)) + item.Item2.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.RomOfKey, mapping[item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!]); + + // Update sampleof + if (!string.IsNullOrEmpty(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)) && mapping.ContainsKey(item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!)) + item.Item2.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.SampleOfKey, mapping[item.Item2.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!]); + + // Add the new item to the output list + newItems.Add(item); + } + + // Replace the old list of roms with the new one + _buckets[key] = newItems.Select(i => i.Item1).ToConcurrentList(); +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + } + catch (Exception ex) when (!throwOnError) + { + logger.Warning(ex.ToString()); + } + } + /// /// Execute all filters in a filter runner on a single bucket /// diff --git a/SabreTools.Filtering/Cleaner.cs b/SabreTools.Filtering/Cleaner.cs index cfe1ea6c..316392b8 100644 --- a/SabreTools.Filtering/Cleaner.cs +++ b/SabreTools.Filtering/Cleaner.cs @@ -123,7 +123,10 @@ namespace SabreTools.Filtering // Process description to machine name if (DescriptionAsName == true) - MachineDescriptionToName(datFile); + { + datFile.Items.MachineDescriptionToName(throwOnError); + datFile.ItemsDB.MachineDescriptionToName(throwOnError); + } // If we are removing scene dates, do that now if (SceneDateStrip == true) @@ -228,109 +231,6 @@ namespace SabreTools.Filtering } } - /// - /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof - /// - /// Current DatFile object to run operations on - /// True if the error that is thrown should be thrown back to the caller, false otherwise - internal void MachineDescriptionToName(DatFile datFile, bool throwOnError = false) - { - try - { - // First we want to get a mapping for all games to description -#if NET40_OR_GREATER || NETCOREAPP - ConcurrentDictionary concurrentDictionary = new(); - ConcurrentDictionary mapping = concurrentDictionary; -#else - Dictionary concurrentDictionary = []; - Dictionary mapping = concurrentDictionary; -#endif -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(datFile.Items.Keys, key => -#else - foreach (var key in datFile.Items.Keys) -#endif - { - var items = datFile.Items[key]; - if (items == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - foreach (DatItem item in items) - { - // If the key mapping doesn't exist, add it -#if NET40_OR_GREATER || NETCOREAPP - mapping.TryAdd(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!, item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); -#else - mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!] = item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.DescriptionKey)!.Replace('/', '_').Replace("\"", "''").Replace(":", " -"); -#endif - } -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - - // Now we loop through every item and update accordingly -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => -#elif NET40_OR_GREATER - Parallel.ForEach(datFile.Items.Keys, key => -#else - foreach (var key in datFile.Items.Keys) -#endif - { - var items = datFile.Items[key]; - if (items == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif - - ConcurrentList newItems = []; - foreach (DatItem item in items) - { - // Update machine name - if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!)) - item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.NameKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!]); - - // Update cloneof - if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!)) - item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.CloneOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey)!]); - - // Update romof - if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!)) - item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.RomOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.RomOfKey)!]); - - // Update sampleof - if (!string.IsNullOrEmpty(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)) && mapping.ContainsKey(item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!)) - item.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.SampleOfKey, mapping[item.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.SampleOfKey)!]); - - // Add the new item to the output list - newItems.Add(item); - } - - // Replace the old list of roms with the new one - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newItems); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - } - catch (Exception ex) when (!throwOnError) - { - logger.Warning(ex.ToString()); - } - } - /// /// Filter a DAT using 1G1R logic given an ordered set of regions ///