From 23e911ecd4b16ba451e68f9933a7eaa0b0fab7fe Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 10 Dec 2020 12:11:32 -0800 Subject: [PATCH] Move updating to new class --- RombaSharp/Features/Diffdat.cs | 2 +- RombaSharp/Features/EDiffdat.cs | 2 +- ...atFile.Updating.cs => DatTool.Updating.cs} | 138 ++++++++++-------- SabreTools/Features/Update.cs | 16 +- 4 files changed, 85 insertions(+), 73 deletions(-) rename SabreTools.DatFiles/{DatFile.Updating.cs => DatTool.Updating.cs} (72%) diff --git a/RombaSharp/Features/Diffdat.cs b/RombaSharp/Features/Diffdat.cs index 88c119a9..24ada330 100644 --- a/RombaSharp/Features/Diffdat.cs +++ b/RombaSharp/Features/Diffdat.cs @@ -63,7 +63,7 @@ in -old DAT file. Ignores those entries in -old that are not in -new."; // Diff against the new datfile DatFile intDat = DatTool.CreateAndParse(newdat); - datfile.DiffAgainst(intDat, false); + DatTool.DiffAgainst(datfile, intDat, false); DatTool.Write(intDat, outdat); } } diff --git a/RombaSharp/Features/EDiffdat.cs b/RombaSharp/Features/EDiffdat.cs index 4e4e5e1e..b476a182 100644 --- a/RombaSharp/Features/EDiffdat.cs +++ b/RombaSharp/Features/EDiffdat.cs @@ -55,7 +55,7 @@ namespace RombaSharp.Features // Diff against the new datfile DatFile intDat = DatTool.CreateAndParse(newdat); - datfile.DiffAgainst(intDat, false); + DatTool.DiffAgainst(datfile, intDat, false); DatTool.Write(intDat, outdat); } } diff --git a/SabreTools.DatFiles/DatFile.Updating.cs b/SabreTools.DatFiles/DatTool.Updating.cs similarity index 72% rename from SabreTools.DatFiles/DatFile.Updating.cs rename to SabreTools.DatFiles/DatTool.Updating.cs index 491ad10c..0e9ba6ac 100644 --- a/SabreTools.DatFiles/DatFile.Updating.cs +++ b/SabreTools.DatFiles/DatTool.Updating.cs @@ -11,15 +11,16 @@ using SabreTools.Logging; // This file represents all methods related to converting and updating DatFiles namespace SabreTools.DatFiles { - public abstract partial class DatFile + public partial class DatTool { /// /// Replace item values from the base set represented by the current DAT /// + /// Current DatFile object to use for updating /// DatFile to replace the values in /// List of Fields representing what should be updated /// True if descriptions should only be replaced if the game name is the same, false otherwise - public void BaseReplace(DatFile intDat, List updateFields, bool onlySame) + public static void BaseReplace(DatFile datFile, DatFile intDat, List updateFields, bool onlySame) { logger.User($"Replacing items in '{intDat.Header.FileName}' from the base DAT"); @@ -27,7 +28,7 @@ namespace SabreTools.DatFiles if (updateFields.Intersect(DatItem.DatItemFields).Any()) { // For comparison's sake, we want to use CRC as the base bucketing - Items.BucketBy(Field.DatItem_CRC, DedupeType.Full); + datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.Full); intDat.Items.BucketBy(Field.DatItem_CRC, DedupeType.None); // Then we do a hashwise comparison against the base DAT @@ -37,7 +38,7 @@ namespace SabreTools.DatFiles List newDatItems = new List(); foreach (DatItem datItem in datItems) { - List dupes = Items.GetDuplicates(datItem, sorted: true); + List dupes = datFile.Items.GetDuplicates(datItem, sorted: true); DatItem newDatItem = datItem.Clone() as DatItem; // Replace fields from the first duplicate, if we have one @@ -57,7 +58,7 @@ namespace SabreTools.DatFiles if (updateFields.Intersect(DatItem.MachineFields).Any()) { // For comparison's sake, we want to use Machine Name as the base bucketing - Items.BucketBy(Field.Machine_Name, DedupeType.Full); + datFile.Items.BucketBy(Field.Machine_Name, DedupeType.Full); intDat.Items.BucketBy(Field.Machine_Name, DedupeType.None); // Then we do a namewise comparison against the base DAT @@ -68,8 +69,8 @@ namespace SabreTools.DatFiles foreach (DatItem datItem in datItems) { DatItem newDatItem = datItem.Clone() as DatItem; - if (Items.ContainsKey(key) && Items[key].Count() > 0) - newDatItem.Machine.ReplaceFields(Items[key][0].Machine, updateFields, onlySame); + if (datFile.Items.ContainsKey(key) && datFile.Items[key].Count() > 0) + newDatItem.Machine.ReplaceFields(datFile.Items[key][0].Machine, updateFields, onlySame); newDatItems.Add(newDatItem); } @@ -84,15 +85,16 @@ namespace SabreTools.DatFiles /// /// Output diffs against a base set represented by the current DAT /// + /// Current DatFile object to use for updating /// DatFile to replace the values in /// True to diff using games, false to use hashes - public void DiffAgainst(DatFile intDat, bool useGames) + public static void DiffAgainst(DatFile datFile, DatFile intDat, bool useGames) { // For comparison's sake, we want to use a base ordering if (useGames) - Items.BucketBy(Field.Machine_Name, DedupeType.None); + datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None); else - Items.BucketBy(Field.DatItem_CRC, DedupeType.None); + datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.None); logger.User($"Comparing '{intDat.Header.FileName}' to base DAT"); @@ -110,11 +112,11 @@ namespace SabreTools.DatFiles if (useGames) { // If the base DAT doesn't contain the key, keep it - if (!Items.ContainsKey(key)) + if (!datFile.Items.ContainsKey(key)) return; // If the number of items is different, then keep it - if (Items[key].Count != intDat.Items[key].Count) + if (datFile.Items[key].Count != intDat.Items[key].Count) return; // Otherwise, compare by name and hash the remaining files @@ -122,7 +124,7 @@ namespace SabreTools.DatFiles foreach (DatItem item in intDat.Items[key]) { // TODO: Make this granular to name as well - if (!Items[key].Contains(item)) + if (!datFile.Items[key].Contains(item)) { exactMatch = false; break; @@ -141,7 +143,7 @@ namespace SabreTools.DatFiles List keepDatItems = new List(); foreach (DatItem datItem in datItems) { - if (!Items.HasDuplicates(datItem, true)) + if (!datFile.Items.HasDuplicates(datItem, true)) keepDatItems.Add(datItem); } @@ -155,15 +157,16 @@ namespace SabreTools.DatFiles /// /// Output cascading diffs /// + /// Current DatFile object to use for updating /// Dat headers used optionally /// List of DatFiles representing the individually indexed items - public List DiffCascade(List datHeaders) + public static List DiffCascade(DatFile datFile, List datHeaders) { // Create a list of DatData objects representing output files List outDats = new List(); // Ensure the current DatFile is sorted optimally - Items.BucketBy(Field.DatItem_CRC, DedupeType.None); + datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.None); // Loop through each of the inputs and get or create a new DatData object InternalStopwatch watch = new InternalStopwatch("Initializing and filling all output DATs"); @@ -172,9 +175,9 @@ namespace SabreTools.DatFiles DatFile[] outDatsArray = new DatFile[datHeaders.Count]; Parallel.For(0, datHeaders.Count, Globals.ParallelOptions, j => { - DatFile diffData = Create(datHeaders[j]); + DatFile diffData = DatFile.Create(datHeaders[j]); diffData.Items = new ItemDictionary(); - FillWithSourceIndex(diffData, j); + FillWithSourceIndex(datFile, diffData, j); outDatsArray[j] = diffData; }); @@ -187,33 +190,35 @@ namespace SabreTools.DatFiles /// /// Output duplicate item diff /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public DatFile DiffDuplicates(List inputs) + public static DatFile DiffDuplicates(DatFile datFile, List inputs) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - return DiffDuplicates(paths); + return DiffDuplicates(datFile, paths); } /// /// Output duplicate item diff /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public DatFile DiffDuplicates(List inputs) + public static DatFile DiffDuplicates(DatFile datFile, List inputs) { InternalStopwatch watch = new InternalStopwatch("Initializing duplicate DAT"); // Fill in any information not in the base DAT - if (string.IsNullOrWhiteSpace(Header.FileName)) - Header.FileName = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.FileName)) + datFile.Header.FileName = "All DATs"; - if (string.IsNullOrWhiteSpace(Header.Name)) - Header.Name = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Name)) + datFile.Header.Name = "datFile.All DATs"; - if (string.IsNullOrWhiteSpace(Header.Description)) - Header.Description = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Description)) + datFile.Header.Description = "datFile.All DATs"; string post = " (Duplicates)"; - DatFile dupeData = Create(Header); + DatFile dupeData = DatFile.Create(datFile.Header); dupeData.Header.FileName += post; dupeData.Header.Name += post; dupeData.Header.Description += post; @@ -224,9 +229,9 @@ namespace SabreTools.DatFiles // Now, loop through the dictionary and populate the correct DATs watch.Start("Populating duplicate DAT"); - Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(Items[key]); + List items = DatItem.Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -253,30 +258,32 @@ namespace SabreTools.DatFiles /// /// Output non-cascading diffs /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public List DiffIndividuals(List inputs) + public static List DiffIndividuals(DatFile datFile, List inputs) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - return DiffIndividuals(paths); + return DiffIndividuals(datFile, paths); } /// /// Output non-cascading diffs /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public List DiffIndividuals(List inputs) + public static List DiffIndividuals(DatFile datFile, List inputs) { InternalStopwatch watch = new InternalStopwatch("Initializing all individual DATs"); // Fill in any information not in the base DAT - if (string.IsNullOrWhiteSpace(Header.FileName)) - Header.FileName = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.FileName)) + datFile.Header.FileName = "All DATs"; - if (string.IsNullOrWhiteSpace(Header.Name)) - Header.Name = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Name)) + datFile.Header.Name = "All DATs"; - if (string.IsNullOrWhiteSpace(Header.Description)) - Header.Description = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Description)) + datFile.Header.Description = "All DATs"; // Loop through each of the inputs and get or create a new DatData object DatFile[] outDatsArray = new DatFile[inputs.Count]; @@ -284,7 +291,7 @@ namespace SabreTools.DatFiles Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => { string innerpost = $" ({j} - {inputs[j].GetNormalizedFileName(true)} Only)"; - DatFile diffData = Create(Header); + DatFile diffData = DatFile.Create(datFile.Header); diffData.Header.FileName += innerpost; diffData.Header.Name += innerpost; diffData.Header.Description += innerpost; @@ -300,9 +307,9 @@ namespace SabreTools.DatFiles // Now, loop through the dictionary and populate the correct DATs watch.Start("Populating all individual DATs"); - Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(Items[key]); + List items = DatItem.Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -324,33 +331,35 @@ namespace SabreTools.DatFiles /// /// Output non-duplicate item diff /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public DatFile DiffNoDuplicates(List inputs) + public static DatFile DiffNoDuplicates(DatFile datFile, List inputs) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - return DiffNoDuplicates(paths); + return DiffNoDuplicates(datFile, paths); } /// /// Output non-duplicate item diff /// + /// Current DatFile object to use for updating /// List of inputs to write out from - public DatFile DiffNoDuplicates(List inputs) + public static DatFile DiffNoDuplicates(DatFile datFile, List inputs) { InternalStopwatch watch = new InternalStopwatch("Initializing no duplicate DAT"); // Fill in any information not in the base DAT - if (string.IsNullOrWhiteSpace(Header.FileName)) - Header.FileName = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.FileName)) + datFile.Header.FileName = "All DATs"; - if (string.IsNullOrWhiteSpace(Header.Name)) - Header.Name = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Name)) + datFile.Header.Name = "All DATs"; - if (string.IsNullOrWhiteSpace(Header.Description)) - Header.Description = "All DATs"; + if (string.IsNullOrWhiteSpace(datFile.Header.Description)) + datFile.Header.Description = "All DATs"; string post = " (No Duplicates)"; - DatFile outerDiffData = Create(Header); + DatFile outerDiffData = DatFile.Create(datFile.Header); outerDiffData.Header.FileName += post; outerDiffData.Header.Name += post; outerDiffData.Header.Description += post; @@ -361,9 +370,9 @@ namespace SabreTools.DatFiles // Now, loop through the dictionary and populate the correct DATs watch.Start("Populating no duplicate DAT"); - Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(Items[key]); + List items = DatItem.Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -389,15 +398,16 @@ namespace SabreTools.DatFiles /// /// Fill a DatFile with all items with a particular source index ID /// + /// Current DatFile object to use for updating /// DatFile to add found items to /// Source index ID to retrieve items for /// DatFile containing all items with the source index ID/returns> - public void FillWithSourceIndex(DatFile indexDat, int index) + public static void FillWithSourceIndex(DatFile datFile, DatFile indexDat, int index) { // Loop through and add the items for this index to the output - Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { - List items = DatItem.Merge(Items[key]); + List items = DatItem.Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -414,20 +424,22 @@ namespace SabreTools.DatFiles /// /// Populate from multiple paths while returning the invividual headers /// + /// Current DatFile object to use for updating /// Paths to DATs to parse /// List of DatHeader objects representing headers - public List PopulateUserData(List inputs) + public static List PopulateUserData(DatFile datFile, List inputs) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - return PopulateUserData(paths); + return PopulateUserData(datFile, paths); } /// /// Populate from multiple paths while returning the invividual headers /// + /// Current DatFile object to use for updating /// Paths to DATs to parse /// List of DatHeader objects representing headers - public List PopulateUserData(List inputs) + public static List PopulateUserData(DatFile datFile, List inputs) { DatFile[] datFiles = new DatFile[inputs.Count]; InternalStopwatch watch = new InternalStopwatch("Processing individual DATs"); @@ -437,8 +449,8 @@ namespace SabreTools.DatFiles { var input = inputs[i]; logger.User($"Adding DAT: {input.CurrentPath}"); - datFiles[i] = Create(Header.CloneFiltering()); - DatTool.ParseInto(datFiles[i], input, i, keep: true); + datFiles[i] = DatFile.Create(datFile.Header.CloneFiltering()); + ParseInto(datFiles[i], input, i, keep: true); }); watch.Stop(); @@ -446,7 +458,7 @@ namespace SabreTools.DatFiles watch.Start("Populating internal DAT"); for (int i = 0; i < inputs.Count; i++) { - AddFromExisting(datFiles[i], true); + datFile.AddFromExisting(datFiles[i], true); } watch.Stop(); diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index 8348909c..538bd762 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -200,9 +200,9 @@ namespace SabreTools.Features // Populate using the correct set List datHeaders; if (updateMode.HasFlag(UpdateMode.DiffAgainst) || updateMode.HasFlag(UpdateMode.BaseReplace)) - datHeaders = userInputDat.PopulateUserData(basePaths); + datHeaders = DatTool.PopulateUserData(userInputDat, basePaths); else - datHeaders = userInputDat.PopulateUserData(inputPaths); + datHeaders = DatTool.PopulateUserData(userInputDat, inputPaths); // Perform additional processing steps userInputDat.ApplyExtras(Extras); @@ -213,7 +213,7 @@ namespace SabreTools.Features // Output only DatItems that are duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) { - DatFile dupeData = userInputDat.DiffDuplicates(inputPaths); + DatFile dupeData = DatTool.DiffDuplicates(userInputDat, inputPaths); InternalStopwatch watch = new InternalStopwatch("Outputting duplicate DAT"); DatTool.Write(dupeData, OutputDir, overwrite: false); @@ -223,7 +223,7 @@ namespace SabreTools.Features // Output only DatItems that are not duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffNoDupesOnly)) { - DatFile outerDiffData = userInputDat.DiffNoDuplicates(inputPaths); + DatFile outerDiffData = DatTool.DiffNoDuplicates(userInputDat, inputPaths); InternalStopwatch watch = new InternalStopwatch("Outputting no duplicate DAT"); DatTool.Write(outerDiffData, OutputDir, overwrite: false); @@ -234,7 +234,7 @@ namespace SabreTools.Features if (updateMode.HasFlag(UpdateMode.DiffIndividualsOnly)) { // Get all of the output DatFiles - List datFiles = userInputDat.DiffIndividuals(inputPaths); + List datFiles = DatTool.DiffIndividuals(userInputDat, inputPaths); // Loop through and output the new DatFiles InternalStopwatch watch = new InternalStopwatch("Outputting all individual DATs"); @@ -269,7 +269,7 @@ namespace SabreTools.Features }); // Get all of the output DatFiles - List datFiles = userInputDat.DiffCascade(datHeaders); + List datFiles = DatTool.DiffCascade(userInputDat, datHeaders); // Loop through and output the new DatFiles InternalStopwatch watch = new InternalStopwatch("Outputting all created DATs"); @@ -303,7 +303,7 @@ namespace SabreTools.Features repDat.ApplyCleaning(Cleaner); // Now replace the fields from the base DatFile - userInputDat.DiffAgainst(repDat, GetBoolean(Features, ByGameValue)); + DatTool.DiffAgainst(userInputDat, repDat, GetBoolean(Features, ByGameValue)); // Finally output the diffed DatFile string interOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue)); @@ -328,7 +328,7 @@ namespace SabreTools.Features repDat.ApplyCleaning(Cleaner); // Now replace the fields from the base DatFile - userInputDat.BaseReplace(repDat, updateFields, GetBoolean(features, OnlySameValue)); + DatTool.BaseReplace(userInputDat, repDat, updateFields, GetBoolean(features, OnlySameValue)); // Finally output the replaced DatFile string interOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));