From 3f6553832b8cd1a84bf369040de4615c6a9ea29d Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 17 Mar 2017 23:44:22 -0700 Subject: [PATCH] [DatFile] Make filtering a separate step Making filter "optional" means that it can be done with multithreading, thus reducing the time that any given DAT takes to parse in most cases and improving the actual filtering in the process. --- .../Dats/Partials/DatFile.ConvertUpdate.cs | 89 +++-- ...ile.Bucketing.cs => DatFile.Manipulate.cs} | 83 ++++- .../Dats/Partials/DatFile.Parsers.cs | 345 +++++------------- SabreTools.Helper/SabreTools.Helper.csproj | 2 +- SabreTools/Partials/SabreTools.Inits.cs | 8 +- 5 files changed, 206 insertions(+), 321 deletions(-) rename SabreTools.Helper/Dats/Partials/{DatFile.Bucketing.cs => DatFile.Manipulate.cs} (92%) diff --git a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs index 7854bfad..620ea8ad 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs @@ -111,22 +111,17 @@ namespace SabreTools.Helper.Dats MergeRoms = MergeRoms, }; - datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single, - root, true, clean, descAsName); + datHeaders[i].Parse(input.Split('¬')[0], i, 0, splitType, true, clean, descAsName); }); Globals.Logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); Globals.Logger.User("Populating internal DAT"); - Parallel.For(0, inputs.Count, - Globals.ParallelOptions, - i => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, i => { // Get the list of keys from the DAT List keys = datHeaders[i].Keys.ToList(); - Parallel.ForEach(keys, - Globals.ParallelOptions, - key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Add everything from the key to the internal DAT AddRange(key, datHeaders[i][key]); @@ -142,6 +137,9 @@ namespace SabreTools.Helper.Dats datHeaders[i].Delete(); }); + // Now that we have a merged DAT, filter it + Filter(filter, single, trim, root); + Globals.Logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); return datHeaders.ToList(); @@ -532,52 +530,49 @@ namespace SabreTools.Helper.Dats public void Update(List inputFileNames, string outDir, bool clean, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root) { - Parallel.ForEach(inputFileNames, - Globals.ParallelOptions, - inputFileName => + Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName => + { + // Clean the input string + if (inputFileName != "") { - // Clean the input string - if (inputFileName != "") - { - inputFileName = Path.GetFullPath(inputFileName); - } + inputFileName = Path.GetFullPath(inputFileName); + } - if (File.Exists(inputFileName)) + if (File.Exists(inputFileName)) + { + DatFile innerDatdata = new DatFile(this); + Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\""); + innerDatdata.Parse(inputFileName, 0, 0, splitType, true, clean, descAsName, + keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); + innerDatdata.Filter(filter, trim, single, root); + + // Try to output the file + innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), overwrite: (outDir != "")); + } + else if (Directory.Exists(inputFileName)) + { + inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar; + + List subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList(); + Parallel.ForEach(subFiles, Globals.ParallelOptions, file => { + Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\""); DatFile innerDatdata = new DatFile(this); - Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\""); - innerDatdata.Parse(inputFileName, 0, 0, filter, splitType, trim, single, - root, true, clean, descAsName, + innerDatdata.Parse(file, 0, 0, splitType, true, clean, descAsName, keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); + innerDatdata.Filter(filter, trim, single, root); // Try to output the file - innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), overwrite: (outDir != "")); - } - else if (Directory.Exists(inputFileName)) - { - inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar; - - Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories), - Globals.ParallelOptions, - file => - { - Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\""); - DatFile innerDatdata = new DatFile(this); - innerDatdata.Parse(file, 0, 0, filter, splitType, - trim, single, root, true, clean, descAsName, - keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); - - // Try to output the file - innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), - overwrite: (outDir != "")); - }); - } - else - { - Globals.Logger.Error("I'm sorry but " + inputFileName + " doesn't exist!"); - return; - } - }); + innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), + overwrite: (outDir != "")); + }); + } + else + { + Globals.Logger.Error("I'm sorry but " + inputFileName + " doesn't exist!"); + return; + } + }); } #endregion diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs similarity index 92% rename from SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs rename to SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs index 8aae7d8d..00f050e5 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs @@ -6,13 +6,19 @@ using System.Web; using SabreTools.Helper.Data; +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; +#endif + namespace SabreTools.Helper.Dats { public partial class DatFile { - #region Instance Methods +#region Instance Methods - #region Bucketing [MODULAR DONE] +#region Bucketing [MODULAR DONE] /// /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method @@ -151,9 +157,62 @@ namespace SabreTools.Helper.Dats _files = sortable; } - #endregion +#endregion - #region Merging/Splitting Methods [MODULAR DONE] +#region Filtering + + /// + /// Filter a DAT based on input parameters and modify the items + /// + /// Filter object for passing to the DatItem level + /// True if we are supposed to trim names to NTFS length, false otherwise + /// True if all games should be replaced by '!', false otherwise + /// String representing root directory to compare against for length calculation + public void Filter(Filter filter, bool single, bool trim, string root) + { + // Loop over every key in the dictionary + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => + { + // For every item in the current key + List items = this[key]; + List newitems = new List(); + Parallel.ForEach(items, Globals.ParallelOptions, item => + { + // If the rom passes the filter, include it + if (filter.ItemPasses(item)) + { + // If we are in single game mode, rename all games + if (single) + { + item.Machine.Name = "!"; + } + + // If we are in NTFS trim mode, trim the game name + if (trim) + { + // Windows max name length is 260 + int usableLength = 260 - item.Machine.Name.Length - root.Length; + if (item.Name.Length > usableLength) + { + string ext = Path.GetExtension(item.Name); + item.Name = item.Name.Substring(0, usableLength - ext.Length); + item.Name += ext; + } + } + + newitems.Add(item); + } + }); + + Remove(key); + AddRange(key, newitems); + }); + } + +#endregion + +#region Merging/Splitting Methods [MODULAR DONE] /// /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets @@ -247,9 +306,9 @@ namespace SabreTools.Helper.Dats RemoveTagsFromChild(); } - #endregion +#endregion - #region Merging/Splitting Helper Methods [MODULAR DONE] +#region Merging/Splitting Helper Methods [MODULAR DONE] /// /// Use romof tags to add roms to the children @@ -830,13 +889,13 @@ namespace SabreTools.Helper.Dats } } - #endregion +#endregion - #endregion // Instance Methods +#endregion // Instance Methods - #region Static Methods +#region Static Methods - #region Bucketing [MODULAR DONE] +#region Bucketing [MODULAR DONE] /// /// Take an arbitrarily ordered List and return a Dictionary sorted by Game @@ -893,8 +952,8 @@ namespace SabreTools.Helper.Dats return sortable; } - #endregion +#endregion - #endregion // Static Methods +#endregion // Static Methods } } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs index 92d68d14..df3f1e10 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs @@ -34,11 +34,9 @@ namespace SabreTools.Helper.Dats /// True if descriptions should be used as names, false otherwise (default) /// True if original extension should be kept, false otherwise (default) /// True if tags from the DAT should be used to merge the output, false otherwise (default) - public void Parse(string filename, int sysid, int srcid, - bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false) + public void Parse(string filename, int sysid, int srcid, bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false) { - Parse(filename, sysid, srcid, new Filter(), SplitType.None, false, false, "", - keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags); + Parse(filename, sysid, srcid, SplitType.None, keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags); } /// @@ -46,12 +44,8 @@ namespace SabreTools.Helper.Dats /// /// Name of the file to be parsed /// System ID for the DAT - /// Source ID for the DAT - /// Filter object for passing to the DatItem level + /// Source ID for the DAT> /// Type of the split that should be performed (split, merged, fully merged) - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if descriptions should be used as names, false otherwise (default) @@ -63,14 +57,8 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - // Rom renaming SplitType splitType, - bool trim, - bool single, - string root, // Miscellaneous bool keep = false, @@ -103,44 +91,44 @@ namespace SabreTools.Helper.Dats switch (FileTools.GetDatFormat(filename)) { case DatFormat.AttractMode: - ParseAttractMode(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName); + ParseAttractMode(filename, sysid, srcid, keep, clean, descAsName); break; case DatFormat.ClrMamePro: case DatFormat.DOSCenter: - ParseCMP(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName); + ParseCMP(filename, sysid, srcid, keep, clean, descAsName); break; case DatFormat.CSV: - ParseCSVTSV(filename, sysid, srcid, ',', filter, trim, single, root, keep, clean, descAsName); + ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, descAsName); break; case DatFormat.Logiqx: case DatFormat.OfflineList: case DatFormat.SabreDat: case DatFormat.SoftwareList: - ParseGenericXML(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName); + ParseGenericXML(filename, sysid, srcid, keep, clean, descAsName); break; case DatFormat.RedumpMD5: - ParseRedumpMD5(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpMD5(filename, sysid, srcid, clean); break; case DatFormat.RedumpSFV: - ParseRedumpSFV(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpSFV(filename, sysid, srcid, clean); break; case DatFormat.RedumpSHA1: - ParseRedumpSHA1(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpSHA1(filename, sysid, srcid, clean); break; case DatFormat.RedumpSHA256: - ParseRedumpSHA256(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpSHA256(filename, sysid, srcid, clean); break; case DatFormat.RedumpSHA384: - ParseRedumpSHA384(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpSHA384(filename, sysid, srcid, clean); break; case DatFormat.RedumpSHA512: - ParseRedumpSHA512(filename, sysid, srcid, filter, trim, single, root, clean); + ParseRedumpSHA512(filename, sysid, srcid, clean); break; case DatFormat.RomCenter: - ParseRC(filename, sysid, srcid, filter, trim, single, root, clean, descAsName); + ParseRC(filename, sysid, srcid, clean, descAsName); break; case DatFormat.TSV: - ParseCSVTSV(filename, sysid, srcid, '\t', filter, trim, single, root, keep, clean, descAsName); + ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, descAsName); break; default: return; @@ -198,10 +186,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if descriptions should be used as names, false otherwise (default) @@ -211,14 +195,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool keep, bool clean, @@ -277,7 +253,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -289,10 +265,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if descriptions should be used as names, false otherwise (default) @@ -302,14 +274,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool keep, bool clean, @@ -421,7 +385,7 @@ namespace SabreTools.Helper.Dats // Now process and add the sample key = ""; - ParseAddHelper(item, filter, trim, single, root, clean, out key); + ParseAddHelper(item, clean, out key); continue; } @@ -506,7 +470,7 @@ namespace SabreTools.Helper.Dats // Now process and add the rom key = ""; - ParseAddHelper(item, filter, trim, single, root, clean, out key); + ParseAddHelper(item, clean, out key); continue; } @@ -719,7 +683,7 @@ namespace SabreTools.Helper.Dats // Now process and add the rom key = ""; - ParseAddHelper(item, filter, trim, single, root, clean, out key); + ParseAddHelper(item, clean, out key); } // If the line is anything but a rom or disk and we're in a block @@ -904,10 +868,6 @@ namespace SabreTools.Helper.Dats /// System ID for the DAT /// Source ID for the DAT /// Delimiter for parsing individual lines - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if SL XML names should be kept, false otherwise (default) @@ -918,14 +878,6 @@ namespace SabreTools.Helper.Dats int srcid, char delim, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool keep, bool clean, @@ -1164,7 +1116,7 @@ namespace SabreTools.Helper.Dats }, }; - ParseAddHelper(archive, filter, trim, single, root, clean, out key); + ParseAddHelper(archive, clean, out key); break; case ItemType.BiosSet: BiosSet biosset = new BiosSet() @@ -1178,7 +1130,7 @@ namespace SabreTools.Helper.Dats }, }; - ParseAddHelper(biosset, filter, trim, single, root, clean, out key); + ParseAddHelper(biosset, clean, out key); break; case ItemType.Disk: Disk disk = new Disk() @@ -1199,7 +1151,7 @@ namespace SabreTools.Helper.Dats ItemStatus = status, }; - ParseAddHelper(disk, filter, trim, single, root, clean, out key); + ParseAddHelper(disk, clean, out key); break; case ItemType.Release: Release release = new Release() @@ -1213,7 +1165,7 @@ namespace SabreTools.Helper.Dats }, }; - ParseAddHelper(release, filter, trim, single, root, clean, out key); + ParseAddHelper(release, clean, out key); break; case ItemType.Rom: Rom rom = new Rom() @@ -1236,7 +1188,7 @@ namespace SabreTools.Helper.Dats ItemStatus = status, }; - ParseAddHelper(rom, filter, trim, single, root, clean, out key); + ParseAddHelper(rom, clean, out key); break; case ItemType.Sample: Sample sample = new Sample() @@ -1250,7 +1202,7 @@ namespace SabreTools.Helper.Dats }, }; - ParseAddHelper(sample, filter, trim, single, root, clean, out key); + ParseAddHelper(sample, clean, out key); break; } } @@ -1262,10 +1214,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if SL XML names should be kept, false otherwise (default) @@ -1279,14 +1227,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool keep, bool clean, @@ -1325,7 +1265,7 @@ namespace SabreTools.Helper.Dats Rom rom = new Rom("null", tempgame); // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out key); + ParseAddHelper(rom, clean, out key); } // Regardless, end the current folder @@ -1871,7 +1811,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(olrom, filter, trim, single, root, clean, out key); + ParseAddHelper(olrom, clean, out key); break; // For Software List and MAME listxml only @@ -1973,7 +1913,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(relrom, filter, trim, single, root, clean, out key); + ParseAddHelper(relrom, clean, out key); subreader.Read(); break; @@ -2016,7 +1956,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(biosrom, filter, trim, single, root, clean, out key); + ParseAddHelper(biosrom, clean, out key); subreader.Read(); break; @@ -2044,7 +1984,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(archiverom, filter, trim, single, root, clean, out key); + ParseAddHelper(archiverom, clean, out key); subreader.Read(); break; @@ -2072,7 +2012,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(samplerom, filter, trim, single, root, clean, out key); + ParseAddHelper(samplerom, clean, out key); subreader.Read(); break; @@ -2215,7 +2155,7 @@ namespace SabreTools.Helper.Dats } // Now process and add the rom - ParseAddHelper(inrom, filter, trim, single, root, clean, out key); + ParseAddHelper(inrom, clean, out key); subreader.Read(); break; @@ -2390,7 +2330,7 @@ namespace SabreTools.Helper.Dats } // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out key); + ParseAddHelper(rom, clean, out key); xtr.Read(); break; @@ -2417,10 +2357,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpMD5( // Standard Dat parsing @@ -2428,14 +2364,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2464,7 +2392,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2476,10 +2404,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpSFV( // Standard Dat parsing @@ -2487,14 +2411,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2523,7 +2439,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2535,10 +2451,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpSHA1( // Standard Dat parsing @@ -2546,14 +2458,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2582,7 +2486,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2594,10 +2498,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpSHA256( // Standard Dat parsing @@ -2605,14 +2505,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2641,7 +2533,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2653,10 +2545,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpSHA384( // Standard Dat parsing @@ -2664,14 +2552,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2700,7 +2580,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2712,10 +2592,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) private void ParseRedumpSHA512( // Standard Dat parsing @@ -2723,14 +2599,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean) { @@ -2759,7 +2627,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } sr.Dispose(); @@ -2771,10 +2639,6 @@ namespace SabreTools.Helper.Dats /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation /// True if game names are sanitized, false otherwise (default) /// True if descriptions should be used as names, false otherwise (default) private void ParseRC( @@ -2783,14 +2647,6 @@ namespace SabreTools.Helper.Dats int sysid, int srcid, - // Rom filtering - Filter filter, - - // Rom renaming - bool trim, - bool single, - string root, - // Miscellaneous bool clean, bool descAsName) @@ -2931,7 +2787,7 @@ namespace SabreTools.Helper.Dats }; // Now process and add the rom - ParseAddHelper(rom, filter, trim, single, root, clean, out string key); + ParseAddHelper(rom, clean, out string key); } } } @@ -2943,11 +2799,9 @@ namespace SabreTools.Helper.Dats /// Add a rom to the Dat after checking /// /// Item data to check against - /// Filter object for passing to the DatItem level - /// True if we are supposed to trim names to NTFS length, false otherwise - /// True if all games should be replaced by '!', false otherwise - /// String representing root directory to compare against for length calculation - private void ParseAddHelper(DatItem item, Filter filter, bool trim, bool single, string root, bool clean, out string key) + /// True if the names should be cleaned to WoD standards, false otherwise + /// Output param containing the key for the item + private void ParseAddHelper(DatItem item, bool clean, out string key) { key = ""; @@ -3046,80 +2900,57 @@ namespace SabreTools.Helper.Dats item = itemDisk; } - // If the rom passes the filter, include it - if (filter.ItemPasses(item)) + // Get the key and add statistical data + switch (item.Type) { - // If we are in single game mode, rename all games - if (single) - { - item.Machine.Name = "!"; - } + case ItemType.Archive: + case ItemType.BiosSet: + case ItemType.Release: + case ItemType.Sample: + key = item.Type.ToString(); + break; + case ItemType.Disk: + key = ((Disk)item).MD5; - // If we are in NTFS trim mode, trim the game name - if (trim) - { - // Windows max name length is 260 - int usableLength = 260 - item.Machine.Name.Length - root.Length; - if (item.Name.Length > usableLength) + // Add statistical data + DiskCount += 1; + if (((Disk)item).ItemStatus != ItemStatus.Nodump) { - string ext = Path.GetExtension(item.Name); - item.Name = item.Name.Substring(0, usableLength - ext.Length); - item.Name += ext; + TotalSize += 0; + MD5Count += (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1); + SHA1Count += (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1); + SHA256Count += (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1); + SHA384Count += (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1); + SHA512Count += (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1); } - } + BaddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); + NodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); + break; + case ItemType.Rom: + key = ((Rom)item).Size + "-" + ((Rom)item).CRC; - // Get the key and add statistical data - switch (item.Type) - { - case ItemType.Archive: - case ItemType.BiosSet: - case ItemType.Release: - case ItemType.Sample: - key = item.Type.ToString(); - break; - case ItemType.Disk: - key = ((Disk)item).MD5; - - // Add statistical data - DiskCount += 1; - if (((Disk)item).ItemStatus != ItemStatus.Nodump) - { - TotalSize += 0; - MD5Count += (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1); - SHA1Count += (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1); - SHA256Count += (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1); - SHA384Count += (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1); - SHA512Count += (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1); - } - BaddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); - NodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); - break; - case ItemType.Rom: - key = ((Rom)item).Size + "-" + ((Rom)item).CRC; - - // Add statistical data - RomCount += 1; - if (((Rom)item).ItemStatus != ItemStatus.Nodump) - { - TotalSize += ((Rom)item).Size; - CRCCount += (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1); - MD5Count += (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1); - SHA1Count += (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1); - SHA256Count += (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1); - SHA384Count += (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1); - SHA512Count += (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1); - } - BaddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); - NodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); - break; - default: - key = "default"; - break; - } - - // Add the item to the DAT - Add(key, item); + // Add statistical data + RomCount += 1; + if (((Rom)item).ItemStatus != ItemStatus.Nodump) + { + TotalSize += ((Rom)item).Size; + CRCCount += (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1); + MD5Count += (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1); + SHA1Count += (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1); + SHA256Count += (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1); + SHA384Count += (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1); + SHA512Count += (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1); + } + BaddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); + NodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); + break; + default: + key = "default"; + break; } + + // Add the item to the DAT + Add(key, item); } #endregion diff --git a/SabreTools.Helper/SabreTools.Helper.csproj b/SabreTools.Helper/SabreTools.Helper.csproj index c13efed4..af8783d0 100644 --- a/SabreTools.Helper/SabreTools.Helper.csproj +++ b/SabreTools.Helper/SabreTools.Helper.csproj @@ -114,7 +114,7 @@ - + diff --git a/SabreTools/Partials/SabreTools.Inits.cs b/SabreTools/Partials/SabreTools.Inits.cs index 2aa564f5..18678613 100644 --- a/SabreTools/Partials/SabreTools.Inits.cs +++ b/SabreTools/Partials/SabreTools.Inits.cs @@ -325,7 +325,7 @@ namespace SabreTools DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { - datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true); + datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -361,7 +361,7 @@ namespace SabreTools DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { - datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true); + datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -728,7 +728,7 @@ namespace SabreTools DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { - datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true); + datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -756,7 +756,7 @@ namespace SabreTools DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { - datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true); + datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));