diff --git a/SabreTools.DatFiles/DatFileTool.cs b/SabreTools.DatFiles/DatFileTool.cs index c5982354..1262183d 100644 --- a/SabreTools.DatFiles/DatFileTool.cs +++ b/SabreTools.DatFiles/DatFileTool.cs @@ -1,13 +1,8 @@ -using System; using System.Collections.Generic; -#if NET40_OR_GREATER || NETCOREAPP -using System.Threading.Tasks; -#endif using SabreTools.Core.Tools; using SabreTools.DatFiles.Formats; using SabreTools.DatItems; using SabreTools.DatItems.Formats; -using SabreTools.IO; using SabreTools.IO.Logging; namespace SabreTools.DatFiles @@ -192,150 +187,5 @@ namespace SabreTools.DatFiles } #endregion - - #region Population - - /// - /// Populate from multiple paths while returning the invividual headers - /// - /// Current DatFile object to use for updating - /// Paths to DATs to parse - /// List of DatHeader objects representing headers - public static List PopulateUserData(DatFile datFile, List inputs) - { - List paths = inputs.ConvertAll(i => new ParentablePath(i)); - return PopulateUserData(datFile, paths); - } - - /// - /// Populate from multiple paths while returning the invividual headers - /// - /// Current DatFile object to use for updating - /// Paths to DATs to parse - /// List of DatHeader objects representing headers - public static List PopulateUserData(DatFile datFile, List inputs) - { - DatFile[] datFiles = new DatFile[inputs.Count]; - InternalStopwatch watch = new("Processing individual DATs"); - - // Parse all of the DATs into their own DatFiles in the array -#if NET452_OR_GREATER || NETCOREAPP - Parallel.For(0, inputs.Count, Core.Globals.ParallelOptions, i => -#elif NET40_OR_GREATER - Parallel.For(0, inputs.Count, i => -#else - for (int i = 0; i < inputs.Count; i++) -#endif - { - var input = inputs[i]; - _staticLogger.User($"Adding DAT: {input.CurrentPath}"); - datFiles[i] = CreateDatFile(datFile.Header.CloneFormat(), datFile.Modifiers); - Parser.ParseInto(datFiles[i], input.CurrentPath, i, keep: true); -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - - watch.Stop(); - - watch.Start("Populating internal DAT"); - for (int i = 0; i < inputs.Count; i++) - { - AddFromExisting(datFile, datFiles[i], true); - //AddFromExistingDB(datFile, datFiles[i], true); - } - - watch.Stop(); - - return [.. Array.ConvertAll(datFiles, d => d.Header)]; - } - - /// - /// Add items from another DatFile to the existing DatFile - /// - /// DatFile to add to - /// DatFile to add from - /// If items should be deleted from the source DatFile - private static void AddFromExisting(DatFile addTo, DatFile addFrom, bool delete = false) - { - // Get the list of keys from the DAT - foreach (string key in addFrom.Items.SortedKeys) - { - // Add everything from the key to the internal DAT - addFrom.GetItemsForBucket(key).ForEach(item => addTo.AddItem(item, statsOnly: false)); - - // Now remove the key from the source DAT - if (delete) - addFrom.RemoveBucket(key); - } - - // Now remove the file dictionary from the source DAT - if (delete) - addFrom.ResetDictionary(); - } - - /// - /// Add items from another DatFile to the existing DatFile - /// - /// DatFile to add to - /// DatFile to add from - /// If items should be deleted from the source DatFile - private static void AddFromExistingDB(DatFile addTo, DatFile addFrom, bool delete = false) - { - // Get all current items, machines, and mappings - var datItems = addFrom.ItemsDB.GetItems(); - var machines = addFrom.GetMachinesDB(); - var sources = addFrom.ItemsDB.GetSources(); - - // Create mappings from old index to new index - var machineRemapping = new Dictionary(); - var sourceRemapping = new Dictionary(); - - // Loop through and add all sources - foreach (var source in sources) - { - long newSourceIndex = addTo.AddSourceDB(source.Value); - sourceRemapping[source.Key] = newSourceIndex; - } - - // Loop through and add all machines - foreach (var machine in machines) - { - long newMachineIndex = addTo.AddMachineDB(machine.Value); - machineRemapping[machine.Key] = newMachineIndex; - } - - // Loop through and add the items -#if NET452_OR_GREATER || NETCOREAPP - Parallel.ForEach(datItems, Core.Globals.ParallelOptions, item => -#elif NET40_OR_GREATER - Parallel.ForEach(datItems, item => -#else - foreach (var item in datItems) -#endif - { - // Get the machine and source index for this item - long machineIndex = addFrom.ItemsDB.GetMachineForItem(item.Key).Key; - long sourceIndex = addFrom.ItemsDB.GetSourceForItem(item.Key).Key; - - addTo.AddItemDB(item.Value, machineRemapping[machineIndex], sourceRemapping[sourceIndex], statsOnly: false); - - // Now remove the key from the source DAT - if (delete) - addFrom.RemoveItemDB(item.Key); - -#if NET40_OR_GREATER || NETCOREAPP - }); -#else - } -#endif - - // Now remove the file dictionary from the source DAT - if (delete) - addFrom.ResetDictionary(); - } - - #endregion } } diff --git a/SabreTools.DatFiles/Parser.cs b/SabreTools.DatFiles/Parser.cs index a96532f8..0ce1f225 100644 --- a/SabreTools.DatFiles/Parser.cs +++ b/SabreTools.DatFiles/Parser.cs @@ -1,8 +1,13 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; +#if NET40_OR_GREATER || NETCOREAPP +using System.Threading.Tasks; +#endif using SabreTools.Core.Tools; using SabreTools.DatItems; +using SabreTools.IO; using SabreTools.IO.Extensions; using SabreTools.IO.Logging; @@ -92,6 +97,147 @@ namespace SabreTools.DatFiles return datFile; } + /// + /// Populate from multiple paths while returning the invividual headers + /// + /// Current DatFile object to use for updating + /// Paths to DATs to parse + /// List of DatHeader objects representing headers + public static List PopulateUserData(DatFile datFile, List inputs) + { + List paths = inputs.ConvertAll(i => new ParentablePath(i)); + return PopulateUserData(datFile, paths); + } + + /// + /// Populate from multiple paths while returning the invividual headers + /// + /// Current DatFile object to use for updating + /// Paths to DATs to parse + /// List of DatHeader objects representing headers + public static List PopulateUserData(DatFile datFile, List inputs) + { + DatFile[] datFiles = new DatFile[inputs.Count]; + InternalStopwatch watch = new("Processing individual DATs"); + + // Parse all of the DATs into their own DatFiles in the array +#if NET452_OR_GREATER || NETCOREAPP + Parallel.For(0, inputs.Count, Core.Globals.ParallelOptions, i => +#elif NET40_OR_GREATER + Parallel.For(0, inputs.Count, i => +#else + for (int i = 0; i < inputs.Count; i++) +#endif + { + var input = inputs[i]; + _staticLogger.User($"Adding DAT: {input.CurrentPath}"); + datFiles[i] = DatFileTool.CreateDatFile(datFile.Header.CloneFormat(), datFile.Modifiers); + ParseInto(datFiles[i], input.CurrentPath, i, keep: true); +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + watch.Stop(); + + watch.Start("Populating internal DAT"); + for (int i = 0; i < inputs.Count; i++) + { + AddFromExisting(datFile, datFiles[i], true); + //AddFromExistingDB(datFile, datFiles[i], true); + } + + watch.Stop(); + + return [.. Array.ConvertAll(datFiles, d => d.Header)]; + } + + /// + /// Add items from another DatFile to the existing DatFile + /// + /// DatFile to add to + /// DatFile to add from + /// If items should be deleted from the source DatFile + private static void AddFromExisting(DatFile addTo, DatFile addFrom, bool delete = false) + { + // Get the list of keys from the DAT + foreach (string key in addFrom.Items.SortedKeys) + { + // Add everything from the key to the internal DAT + addFrom.GetItemsForBucket(key).ForEach(item => addTo.AddItem(item, statsOnly: false)); + + // Now remove the key from the source DAT + if (delete) + addFrom.RemoveBucket(key); + } + + // Now remove the file dictionary from the source DAT + if (delete) + addFrom.ResetDictionary(); + } + + /// + /// Add items from another DatFile to the existing DatFile + /// + /// DatFile to add to + /// DatFile to add from + /// If items should be deleted from the source DatFile + private static void AddFromExistingDB(DatFile addTo, DatFile addFrom, bool delete = false) + { + // Get all current items, machines, and mappings + var datItems = addFrom.ItemsDB.GetItems(); + var machines = addFrom.GetMachinesDB(); + var sources = addFrom.ItemsDB.GetSources(); + + // Create mappings from old index to new index + var machineRemapping = new Dictionary(); + var sourceRemapping = new Dictionary(); + + // Loop through and add all sources + foreach (var source in sources) + { + long newSourceIndex = addTo.AddSourceDB(source.Value); + sourceRemapping[source.Key] = newSourceIndex; + } + + // Loop through and add all machines + foreach (var machine in machines) + { + long newMachineIndex = addTo.AddMachineDB(machine.Value); + machineRemapping[machine.Key] = newMachineIndex; + } + + // Loop through and add the items +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(datItems, Core.Globals.ParallelOptions, item => +#elif NET40_OR_GREATER + Parallel.ForEach(datItems, item => +#else + foreach (var item in datItems) +#endif + { + // Get the machine and source index for this item + long machineIndex = addFrom.ItemsDB.GetMachineForItem(item.Key).Key; + long sourceIndex = addFrom.ItemsDB.GetSourceForItem(item.Key).Key; + + addTo.AddItemDB(item.Value, machineRemapping[machineIndex], sourceRemapping[sourceIndex], statsOnly: false); + + // Now remove the key from the source DAT + if (delete) + addFrom.RemoveItemDB(item.Key); + +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + // Now remove the file dictionary from the source DAT + if (delete) + addFrom.ResetDictionary(); + } + /// /// Get what type of DAT the input file is /// @@ -107,7 +253,7 @@ namespace SabreTools.DatFiles string? ext = filename.GetNormalizedExtension(); // Check if file exists - if (!File.Exists(filename)) + if (!System.IO.File.Exists(filename)) return 0; // Some formats should only require the extension to know @@ -147,7 +293,7 @@ namespace SabreTools.DatFiles try { - using StreamReader sr = File.OpenText(filename); + using StreamReader sr = System.IO.File.OpenText(filename); first = FindNextLine(sr); second = FindNextLine(sr); } diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index d3597d48..6eb448ce 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -507,9 +507,9 @@ namespace SabreTools.Features #else if (updateMode.HasFlag(UpdateMode.DiffAgainst) || updateMode.HasFlag(UpdateMode.BaseReplace)) #endif - return DatFileTool.PopulateUserData(userInputDat, basePaths); + return Parser.PopulateUserData(userInputDat, basePaths); else - return DatFileTool.PopulateUserData(userInputDat, inputPaths); + return Parser.PopulateUserData(userInputDat, inputPaths); } ///