From 1bf7a94194160aafd842bf129aa491afd2f1d186 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Sat, 18 Mar 2017 16:35:36 -0700 Subject: [PATCH] [DatFile] Make descAsName a separate method Making this separate allows for updating of cloneof/romof/sampleof tags at the same time and in parallel. This change also fully hooks up the remove unicode flag, though the flag itself has some issues. --- .../Dats/Partials/DatFile.ConvertUpdate.cs | 20 +-- .../Dats/Partials/DatFile.Manipulate.cs | 139 ++++++++++-------- .../Dats/Partials/DatFile.Parsers.cs | 57 +++---- SabreTools/Partials/SabreTools.Inits.cs | 4 +- 4 files changed, 108 insertions(+), 112 deletions(-) diff --git a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs index b5385c58..9ed11a6a 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs @@ -32,6 +32,7 @@ namespace SabreTools.Helper.Dats /// True if the first cascaded diff file should be skipped on output, false otherwise /// True if the date should not be appended to the default name, false otherwise [OBSOLETE] /// True to clean the game names to WoD standard, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) /// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default) /// Filter object to be passed to the DatItem level /// Type of the split that should be performed (split, merged, fully merged) @@ -39,7 +40,7 @@ namespace SabreTools.Helper.Dats /// True if all games should be replaced by '!', false otherwise /// String representing root directory to compare against for length calculation public void DetermineUpdateType(List inputPaths, string outDir, bool merge, DiffMode diff, bool inplace, bool skip, - bool bare, bool clean, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root) + bool bare, bool clean, bool remUnicode, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root) { // If we're in merging or diffing mode, use the full list of inputs if (merge || diff != 0) @@ -54,8 +55,8 @@ namespace SabreTools.Helper.Dats } // Create a dictionary of all ROMs from the input DATs - List datHeaders = PopulateUserData(newInputFileNames, inplace, clean, descAsName, - outDir, filter, splitType, trim, single, root); + List datHeaders = PopulateUserData(newInputFileNames, inplace, clean, + remUnicode, descAsName, outDir, filter, splitType, trim, single, root); // Modify the Dictionary if necessary and output the results if (diff != 0 && diff < DiffMode.Cascade) @@ -76,7 +77,7 @@ namespace SabreTools.Helper.Dats // Otherwise, loop through all of the inputs individually else { - Update(inputPaths, outDir, clean, descAsName, filter, splitType, trim, single, root); + Update(inputPaths, outDir, clean, remUnicode, descAsName, filter, splitType, trim, single, root); } return; } @@ -90,8 +91,8 @@ namespace SabreTools.Helper.Dats /// True if all games should be replaced by '!', false otherwise /// String representing root directory to compare against for length calculation /// List of DatData objects representing headers - private List PopulateUserData(List inputs, bool inplace, bool clean, bool descAsName, string outDir, - Filter filter, SplitType splitType, bool trim, bool single, string root) + private List PopulateUserData(List inputs, bool inplace, bool clean, bool remUnicode, bool descAsName, + string outDir, Filter filter, SplitType splitType, bool trim, bool single, string root) { DatFile[] datHeaders = new DatFile[inputs.Count]; DateTime start = DateTime.Now; @@ -518,14 +519,15 @@ namespace SabreTools.Helper.Dats /// True if the first cascaded diff file should be skipped on output, false otherwise /// True if the date should not be appended to the default name, false otherwise [OBSOLETE] /// True to clean the game names to WoD standard, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) /// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default) /// Filter object to be passed to the DatItem level /// Type of the split that should be performed (split, merged, fully merged) /// True if we are supposed to trim names to NTFS length, false otherwise /// True if all games should be replaced by '!', false otherwise /// String representing root directory to compare against for length calculation - public void Update(List inputFileNames, string outDir, bool clean, bool descAsName, Filter filter, - SplitType splitType, bool trim, bool single, string root) + public void Update(List inputFileNames, string outDir, bool clean, bool remUnicode, bool descAsName, + Filter filter, SplitType splitType, bool trim, bool single, string root) { Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName => { @@ -539,7 +541,7 @@ namespace SabreTools.Helper.Dats { DatFile innerDatdata = new DatFile(this); Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\""); - innerDatdata.Parse(inputFileName, 0, 0, splitType, true, clean, descAsName, + innerDatdata.Parse(inputFileName, 0, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName, keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); innerDatdata.Filter(filter, trim, single, root); diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs index 88301df0..d527a269 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs @@ -208,6 +208,82 @@ namespace SabreTools.Helper.Dats }); } + /// + /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof + /// + public void MachineDescriptionToName() + { + try + { + // First we want to get a mapping for all games to description + Dictionary mapping = new Dictionary(); + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + List items = this[key]; + Parallel.ForEach(items, Globals.ParallelOptions, item => + { + lock (mapping) + { + // If the key mapping doesn't exist, add it + if (!mapping.ContainsKey(item.Machine.Name)) + { + mapping.Add(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''")); + } + } + }); + }); + + // Now we loop through every item and update accordingly + keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + List items = this[key]; + List newItems = new List(); + Parallel.ForEach(items, Globals.ParallelOptions, item => + { + // Update machine name + if (item.Machine.Name != null && mapping.ContainsKey(item.Machine.Name)) + { + item.Machine.Name = mapping[item.Machine.Name]; + } + + // Update cloneof + if (item.Machine.CloneOf != null && mapping.ContainsKey(item.Machine.CloneOf)) + { + item.Machine.CloneOf = mapping[item.Machine.CloneOf]; + } + + // Update romof + if (item.Machine.RomOf != null && mapping.ContainsKey(item.Machine.RomOf)) + { + item.Machine.RomOf = mapping[item.Machine.RomOf]; + } + + // Update sampleof + if (item.Machine.SampleOf != null && mapping.ContainsKey(item.Machine.SampleOf)) + { + item.Machine.SampleOf = mapping[item.Machine.SampleOf]; + } + + // Add the new item to the output list + lock (newItems) + { + newItems.Add(item); + } + }); + + // Replace the old list of roms with the new one + Remove(key); + AddRange(key, newItems); + }); + } + catch (Exception ex) + { + Globals.Logger.Warning(ex.ToString()); + } + } + #endregion #region Merging/Splitting Methods @@ -890,68 +966,5 @@ namespace SabreTools.Helper.Dats #endregion #endregion // Instance Methods - -#region Static Methods - -#region Bucketing - - /// - /// Take an arbitrarily ordered List and return a Dictionary sorted by Game - /// - /// Input unsorted list - /// True if roms should be deduped, false otherwise - /// True if games should only be compared on game and file name, false if system and source are counted - /// True if the number of hashes counted is to be output (default), false otherwise - /// SortedDictionary bucketed by game name - public static SortedDictionary> BucketListByGame(List list, bool mergeroms, bool norename, bool output = true) - { - Globals.Logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); - - SortedDictionary> sortable = new SortedDictionary>(); - long count = 0; - - // If we have a null dict or an empty one, output a new dictionary - if (list == null || list.Count == 0) - { - return sortable; - } - - // If we're merging the roms, do so - if (mergeroms) - { - list = DatItem.Merge(list); - } - - // Now add each of the roms to their respective games - foreach (DatItem rom in list) - { - if (rom == null) - { - continue; - } - - count++; - string newkey = (norename ? "" - : rom.SystemID.ToString().PadLeft(10, '0') - + "-" - + rom.SourceID.ToString().PadLeft(10, '0') + "-") - + (rom.Machine == null || String.IsNullOrEmpty(rom.Machine.Name) - ? "Default" - : rom.Machine.Name.ToLowerInvariant()); - newkey = HttpUtility.HtmlEncode(newkey); - - if (!sortable.ContainsKey(newkey)) - { - sortable.Add(newkey, new List()); - } - sortable[newkey].Add(rom); - } - - return sortable; - } - -#endregion - -#endregion // Static Methods } } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs index d70ce8fd..debf97e2 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs @@ -55,8 +55,6 @@ namespace SabreTools.Helper.Dats /// True if descriptions should be used as names, false otherwise (default) /// True if original extension should be kept, false otherwise (default) /// True if tags from the DAT should be used to merge the output, false otherwise (default) - /// TODO: If replacing name with description, is it possible to update cloneof/romof/sampleof tags as well? - /// TODO: If replacing name with description, should this be done in a separate step instead of on parse? It might help the other task public void Parse( // Standard Dat parsing string filename, @@ -98,20 +96,20 @@ namespace SabreTools.Helper.Dats switch (FileTools.GetDatFormat(filename)) { case DatFormat.AttractMode: - ParseAttractMode(filename, sysid, srcid, keep, clean, remUnicode, descAsName); + ParseAttractMode(filename, sysid, srcid, keep, clean, remUnicode); break; case DatFormat.ClrMamePro: case DatFormat.DOSCenter: - ParseCMP(filename, sysid, srcid, keep, clean, remUnicode, descAsName); + ParseCMP(filename, sysid, srcid, keep, clean, remUnicode); break; case DatFormat.CSV: - ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, remUnicode, descAsName); + ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, remUnicode); break; case DatFormat.Logiqx: case DatFormat.OfflineList: case DatFormat.SabreDat: case DatFormat.SoftwareList: - ParseGenericXML(filename, sysid, srcid, keep, clean, remUnicode, descAsName); + ParseGenericXML(filename, sysid, srcid, keep, clean, remUnicode); break; case DatFormat.RedumpMD5: ParseRedumpMD5(filename, sysid, srcid, clean, remUnicode); @@ -132,10 +130,10 @@ namespace SabreTools.Helper.Dats ParseRedumpSHA512(filename, sysid, srcid, clean, remUnicode); break; case DatFormat.RomCenter: - ParseRC(filename, sysid, srcid, clean, remUnicode, descAsName); + ParseRC(filename, sysid, srcid, clean, remUnicode); break; case DatFormat.TSV: - ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, remUnicode, descAsName); + ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, remUnicode); break; default: return; @@ -146,6 +144,12 @@ namespace SabreTools.Helper.Dats Globals.Logger.Error("Error with file '" + filename + "': " + ex.ToString()); } + // If we want to use descriptions as names, update everything + if (descAsName) + { + MachineDescriptionToName(); + } + // If we are using tags from the DAT, set the proper input for split type unless overridden if (useTags && splitType == SplitType.None) { @@ -196,7 +200,6 @@ namespace SabreTools.Helper.Dats /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) - /// True if descriptions should be used as names, false otherwise (default) private void ParseAttractMode( // Standard Dat parsing string filename, @@ -206,8 +209,7 @@ namespace SabreTools.Helper.Dats // Miscellaneous bool keep, bool clean, - bool remUnicode, - bool descAsName) + bool remUnicode) { // Open a file reader Encoding enc = Style.GetEncoding(filename); @@ -252,7 +254,7 @@ namespace SabreTools.Helper.Dats Machine = new Machine { - Name = (descAsName ? gameinfo[1] : gameinfo[0]), + Name = gameinfo[0], Description = gameinfo[1], CloneOf = gameinfo[3], Year = gameinfo[4], @@ -277,7 +279,6 @@ namespace SabreTools.Helper.Dats /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) - /// True if descriptions should be used as names, false otherwise (default) private void ParseCMP( // Standard Dat parsing string filename, @@ -287,8 +288,7 @@ namespace SabreTools.Helper.Dats // Miscellaneous bool keep, bool clean, - bool remUnicode, - bool descAsName) + bool remUnicode) { // Open a file reader Encoding enc = Style.GetEncoding(filename); @@ -712,12 +712,6 @@ namespace SabreTools.Helper.Dats break; case "description": gamedesc = itemval; - - // If we want to have the description as the name, do so - if (descAsName) - { - tempgamename = gamedesc.Replace('/', '_').Replace("\"", "''"); - } break; case "romof": romof = itemval; @@ -882,7 +876,6 @@ namespace SabreTools.Helper.Dats /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) - /// True if SL XML names should be kept, false otherwise (default) private void ParseCSVTSV( // Standard Dat parsing string filename, @@ -893,8 +886,7 @@ namespace SabreTools.Helper.Dats // Miscellaneous bool keep, bool clean, - bool remUnicode, - bool descAsName) + bool remUnicode) { // Open a file reader Encoding enc = Style.GetEncoding(filename); @@ -1035,7 +1027,6 @@ namespace SabreTools.Helper.Dats break; case "Machine.Description": machineDesc = value; - machineName = (descAsName ? value : machineName); break; case "DatItem.Type": switch (value.ToLowerInvariant()) @@ -1230,7 +1221,6 @@ namespace SabreTools.Helper.Dats /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) - /// True if SL XML names should be kept, false otherwise (default) /// /// TODO: Software Lists - sharedfeat tag (read-in, write-out) /// @@ -1243,8 +1233,7 @@ namespace SabreTools.Helper.Dats // Miscellaneous bool keep, bool clean, - bool remUnicode, - bool descAsName) + bool remUnicode) { // Prepare all internal variables XmlReader subreader, headreader, flagreader; @@ -1877,12 +1866,6 @@ namespace SabreTools.Helper.Dats // For Logiqx, SabreDAT, and Software List case "description": machine.Description = subreader.ReadElementContentAsString(); - - // If we want to have the description as the name, do so - if (descAsName) - { - machine.Name = machine.Description.Replace('/', '_').Replace("\"", "''"); - } break; case "year": machine.Year = subreader.ReadElementContentAsString(); @@ -2667,7 +2650,6 @@ namespace SabreTools.Helper.Dats /// Source ID for the DAT /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) - /// True if descriptions should be used as names, false otherwise (default) private void ParseRC( // Standard Dat parsing string filename, @@ -2676,8 +2658,7 @@ namespace SabreTools.Helper.Dats // Miscellaneous bool clean, - bool remUnicode, - bool descAsName) + bool remUnicode) { // Open a file reader Encoding enc = Style.GetEncoding(filename); @@ -2804,7 +2785,7 @@ namespace SabreTools.Helper.Dats Machine = new Machine { - Name = (descAsName ? rominfo[4] : rominfo[3]), + Name = rominfo[3], Description = rominfo[4], CloneOf = rominfo[1], RomOf = rominfo[8], diff --git a/SabreTools/Partials/SabreTools.Inits.cs b/SabreTools/Partials/SabreTools.Inits.cs index a4e74ada..fb3bb722 100644 --- a/SabreTools/Partials/SabreTools.Inits.cs +++ b/SabreTools/Partials/SabreTools.Inits.cs @@ -700,8 +700,8 @@ namespace SabreTools Romba = romba, }; - userInputDat.DetermineUpdateType(inputs, outDir, merge, diffMode, inplace, skip, bare, clean, descAsName, - filter, splitType, trim, single, root); + userInputDat.DetermineUpdateType(inputs, outDir, merge, diffMode, inplace, skip, bare, clean, + remUnicode, descAsName, filter, splitType, trim, single, root); } ///