From 98bbbfa5d998cb9d7be5575c64c5bfd82353ef23 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 1 Mar 2017 20:28:32 -0800 Subject: [PATCH] [DatFile] Proliferate multithreading a little more --- RombaSharp/Partials/RombaSharp.Helpers.cs | 6 +-- SabreTools.Helper/Dats/DatItem.cs | 38 +++++++------- .../Dats/Partials/DatFile.Bucketing.cs | 25 +++++---- .../Dats/Partials/DatFile.ConvertUpdate.cs | 37 +++++++------ .../Dats/Partials/DatFile.Parsers.cs | 12 +++-- .../Dats/Partials/DatFile.Rebuild.cs | 46 +++++++++------- .../Dats/Partials/DatFile.Writers.cs | 6 ++- SabreTools.Helper/Tools/ArchiveTools.cs | 52 ++++++++----------- 8 files changed, 119 insertions(+), 103 deletions(-) diff --git a/RombaSharp/Partials/RombaSharp.Helpers.cs b/RombaSharp/Partials/RombaSharp.Helpers.cs index fa6001c8..6c36e120 100644 --- a/RombaSharp/Partials/RombaSharp.Helpers.cs +++ b/RombaSharp/Partials/RombaSharp.Helpers.cs @@ -405,7 +405,7 @@ namespace RombaSharp // First get a list of SHA-1's from the input DATs DatFile datroot = new DatFile { Type = "SuperDAT", }; datroot.PopulateFromDir(_dats, Hash.SHA256 & Hash.SHA384 & Hash.SHA512, false, false, false, false, false, _tmpdir, false, null, 4, _logger); - datroot.BucketBy(SortedBy.SHA1, false /* mergeroms */, _logger); + datroot.BucketBy(SortedBy.SHA1, false /* mergeroms */, 4 /* maxDegreeOfParallelism */, _logger); // Create a List of dat hashes in the database (SHA-1) List databaseDats = new List(); @@ -435,7 +435,7 @@ namespace RombaSharp unneeded.Add(hash); } } - datroot.BucketBy(SortedBy.Game, false /* mergeroms */, _logger, norename: true); + datroot.BucketBy(SortedBy.Game, false /* mergeroms */, 4 /* maxDegreeOfParallelism */, _logger, norename: true); _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -619,7 +619,7 @@ namespace RombaSharp // Now rescan the depot itself DatFile depot = new DatFile(); depot.PopulateFromDir(depotname, Hash.SHA256 & Hash.SHA384 & Hash.SHA512, false, false, true, false, false, _tmpdir, false, null, _workers, _logger); - depot.BucketBy(SortedBy.SHA1, false /* mergeroms */, _logger); + depot.BucketBy(SortedBy.SHA1, false /* mergeroms */, 4 /* maxDegreeOfParallelism */, _logger); // Set the base queries to use string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; diff --git a/SabreTools.Helper/Dats/DatItem.cs b/SabreTools.Helper/Dats/DatItem.cs index af4ebee5..3dbb8bd8 100644 --- a/SabreTools.Helper/Dats/DatItem.cs +++ b/SabreTools.Helper/Dats/DatItem.cs @@ -243,9 +243,10 @@ namespace SabreTools.Helper.Dats /// Check if a DAT contains the given rom /// /// Dat to match against + /// Integer representing the maximum amount of parallelization to be used /// Logger object for console and/or file output /// True if it contains the rom, false otherwise - public bool HasDuplicates(DatFile datdata, Logger logger) + public bool HasDuplicates(DatFile datdata, int maxDegreeOfParallelism, Logger logger) { // Check for an empty rom list first if (datdata.Count == 0) @@ -254,7 +255,7 @@ namespace SabreTools.Helper.Dats } // We want to get the proper key for the DatItem - string key = SortAndGetKey(datdata, logger); + string key = SortAndGetKey(datdata, maxDegreeOfParallelism, logger); // If the key doesn't exist, return the empty list if (!datdata.ContainsKey(key)) @@ -280,10 +281,11 @@ namespace SabreTools.Helper.Dats /// List all duplicates found in a DAT based on a rom /// /// Dat to match against + /// Integer representing the maximum amount of parallelization to be used /// Logger object for console and/or file output /// True to remove matched roms from the input, false otherwise (default) /// List of matched DatItem objects - public List GetDuplicates(DatFile datdata, Logger logger, bool remove = false) + public List GetDuplicates(DatFile datdata, int maxDegreeOfParallelism, Logger logger, bool remove = false) { List output = new List(); @@ -294,7 +296,7 @@ namespace SabreTools.Helper.Dats } // We want to get the proper key for the DatItem - string key = SortAndGetKey(datdata, logger); + string key = SortAndGetKey(datdata, maxDegreeOfParallelism, logger); // If the key doesn't exist, return the empty list if (!datdata.ContainsKey(key)) @@ -333,7 +335,7 @@ namespace SabreTools.Helper.Dats /// Dat to match against /// Logger object for console and/or file output /// Key to try to use - private string SortAndGetKey(DatFile datdata, Logger logger) + private string SortAndGetKey(DatFile datdata, int maxDegreeOfParallelism, Logger logger) { string key = null; @@ -345,12 +347,12 @@ namespace SabreTools.Helper.Dats if (_itemType == ItemType.Rom) { key = ((Rom)this).SHA512; - datdata.BucketBy(SortedBy.SHA512, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA512, false /* mergeroms */, maxDegreeOfParallelism, logger); } else { key = ((Disk)this).SHA512; - datdata.BucketBy(SortedBy.SHA512, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA512, false /* mergeroms */, maxDegreeOfParallelism, logger); } } @@ -362,12 +364,12 @@ namespace SabreTools.Helper.Dats if (_itemType == ItemType.Rom) { key = ((Rom)this).SHA384; - datdata.BucketBy(SortedBy.SHA384, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA384, false /* mergeroms */, maxDegreeOfParallelism, logger); } else { key = ((Disk)this).SHA384; - datdata.BucketBy(SortedBy.SHA384, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA384, false /* mergeroms */, maxDegreeOfParallelism, logger); } } @@ -379,12 +381,12 @@ namespace SabreTools.Helper.Dats if (_itemType == ItemType.Rom) { key = ((Rom)this).SHA256; - datdata.BucketBy(SortedBy.SHA256, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA256, false /* mergeroms */, maxDegreeOfParallelism, logger); } else { key = ((Disk)this).SHA256; - datdata.BucketBy(SortedBy.SHA256, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA256, false /* mergeroms */, maxDegreeOfParallelism, logger); } } @@ -396,12 +398,12 @@ namespace SabreTools.Helper.Dats if (_itemType == ItemType.Rom) { key = ((Rom)this).SHA1; - datdata.BucketBy(SortedBy.SHA1, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA1, false /* mergeroms */, maxDegreeOfParallelism, logger); } else { key = ((Disk)this).SHA1; - datdata.BucketBy(SortedBy.SHA1, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.SHA1, false /* mergeroms */, maxDegreeOfParallelism, logger); } } @@ -413,12 +415,12 @@ namespace SabreTools.Helper.Dats if (_itemType == ItemType.Rom) { key = ((Rom)this).MD5; - datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, maxDegreeOfParallelism, logger); } else { key = ((Disk)this).MD5; - datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, maxDegreeOfParallelism, logger); } } @@ -426,21 +428,21 @@ namespace SabreTools.Helper.Dats else if (_itemType == ItemType.Disk) { key = ((Disk)this).MD5; - datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.MD5, false /* mergeroms */, maxDegreeOfParallelism, logger); } // If we've gotten here and we have a Rom, sort by CRC else if (_itemType == ItemType.Rom) { key = ((Rom)this).CRC; - datdata.BucketBy(SortedBy.CRC, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.CRC, false /* mergeroms */, maxDegreeOfParallelism, logger); } // Otherwise, we use -1 as the key else { key = "-1"; - datdata.BucketBy(SortedBy.Size, false /* mergeroms */, logger); + datdata.BucketBy(SortedBy.Size, false /* mergeroms */, maxDegreeOfParallelism, logger); } return key; diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs b/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs index e17be177..63dcdba2 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs @@ -19,10 +19,11 @@ namespace SabreTools.Helper.Dats /// /// SortedBy enum representing how to sort the individual items /// True if roms should be deduped, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted - public void BucketBy(SortedBy bucketBy, bool mergeroms, Logger logger, bool lower = true, bool norename = true) + public void BucketBy(SortedBy bucketBy, bool mergeroms, int maxDegreeOfParallelism, Logger logger, bool lower = true, bool norename = true) { // If we already have the right sorting, trust it if (_sortedBy == bucketBy) @@ -41,6 +42,7 @@ namespace SabreTools.Helper.Dats // First do the initial sort of all of the roms List keys = Keys.ToList(); Parallel.ForEach(keys, + new ParallelOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism }, key => { List roms = this[key]; @@ -129,6 +131,7 @@ namespace SabreTools.Helper.Dats // Now go through and sort all of the individual lists keys = sortable.Keys.ToList(); Parallel.ForEach(keys, + new ParallelOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism }, key => { List sortedlist = sortable[key]; @@ -152,13 +155,14 @@ namespace SabreTools.Helper.Dats /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets /// /// True if roms should be deduped, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output - public void CreateFullyNonMergedSets(bool mergeroms, Logger logger) + public void CreateFullyNonMergedSets(bool mergeroms, int maxDegreeOfParallelism, Logger logger) { logger.User("Creating fully non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game - BucketBy(SortedBy.Game, mergeroms, logger, norename: true); + BucketBy(SortedBy.Game, mergeroms, maxDegreeOfParallelism, logger, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information @@ -179,13 +183,14 @@ namespace SabreTools.Helper.Dats /// Use cloneof tags to create merged sets and remove the tags /// /// True if roms should be deduped, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output - public void CreateMergedSets(bool mergeroms, Logger logger) + public void CreateMergedSets(bool mergeroms, int maxDegreeOfParallelism, Logger logger) { logger.User("Creating merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game - BucketBy(SortedBy.Game, mergeroms, logger, norename: true); + BucketBy(SortedBy.Game, mergeroms, maxDegreeOfParallelism, logger, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information @@ -202,13 +207,14 @@ namespace SabreTools.Helper.Dats /// Use cloneof tags to create non-merged sets and remove the tags /// /// True if roms should be deduped, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output - public void CreateNonMergedSets(bool mergeroms, Logger logger) + public void CreateNonMergedSets(bool mergeroms, int maxDegreeOfParallelism, Logger logger) { logger.User("Creating non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game - BucketBy(SortedBy.Game, mergeroms, logger, norename: true); + BucketBy(SortedBy.Game, mergeroms, maxDegreeOfParallelism, logger, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information @@ -225,13 +231,14 @@ namespace SabreTools.Helper.Dats /// Use cloneof and romof tags to create split sets and remove the tags /// /// True if roms should be deduped, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output - public void CreateSplitSets(bool mergeroms, Logger logger) + public void CreateSplitSets(bool mergeroms, int maxDegreeOfParallelism, Logger logger) { logger.User("Creating split sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game - BucketBy(SortedBy.Game, mergeroms, logger, norename: true); + BucketBy(SortedBy.Game, mergeroms, maxDegreeOfParallelism, logger, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information diff --git a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs index 5991f704..0af83528 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs @@ -66,17 +66,17 @@ namespace SabreTools.Helper.Dats // Modify the Dictionary if necessary and output the results if (diff != 0 && diff < DiffMode.Cascade) { - DiffNoCascade(diff, outDir, newInputFileNames, logger); + DiffNoCascade(diff, outDir, newInputFileNames, maxDegreeOfParallelism, logger); } // If we're in cascade and diff, output only cascaded diffs else if (diff != 0 && diff >= DiffMode.Cascade) { - DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip, logger); + DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip, maxDegreeOfParallelism, logger); } // Output all entries with user-defined merge else { - MergeNoDiff(outDir, newInputFileNames, datHeaders, logger); + MergeNoDiff(outDir, newInputFileNames, datHeaders, maxDegreeOfParallelism, logger); } } // Otherwise, loop through all of the inputs individually @@ -120,7 +120,8 @@ namespace SabreTools.Helper.Dats MergeRoms = MergeRoms, }; - datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single, root, logger, true, clean, descAsName); + datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single, + root, maxDegreeOfParallelism, logger, true, clean, descAsName); }); logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -148,8 +149,9 @@ namespace SabreTools.Helper.Dats /// Non-zero flag for diffing mode, zero otherwise /// Output directory to write the DATs to /// List of inputs to write out from + /// Integer representing the maximum amount of parallelization to be used /// Logging object for console and file output - public void DiffNoCascade(DiffMode diff, string outDir, List inputs, Logger logger) + public void DiffNoCascade(DiffMode diff, string outDir, List inputs, int maxDegreeOfParallelism, Logger logger) { DateTime start = DateTime.Now; logger.User("Initializing all output DATs"); @@ -275,13 +277,13 @@ namespace SabreTools.Helper.Dats // Output the difflist (a-b)+(b-a) diff if ((diff & DiffMode.NoDupes) != 0) { - outerDiffData.WriteToFile(outDir, logger); + outerDiffData.WriteToFile(outDir, maxDegreeOfParallelism, logger); } // Output the (ab) diff if ((diff & DiffMode.Dupes) != 0) { - dupeData.WriteToFile(outDir, logger); + dupeData.WriteToFile(outDir, maxDegreeOfParallelism, logger); } // Output the individual (a-b) DATs @@ -296,7 +298,7 @@ namespace SabreTools.Helper.Dats : (Path.GetDirectoryName(split[0]).Remove(0, split[1].Length))); // Try to output the file - outDats[j].WriteToFile(path, logger); + outDats[j].WriteToFile(path, maxDegreeOfParallelism, logger); } } logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -310,8 +312,9 @@ namespace SabreTools.Helper.Dats /// List of inputs to write out from /// Dat headers used optionally /// True if the first cascaded diff file should be skipped on output, false otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logging object for console and file output - public void DiffCascade(string outDir, bool inplace, List inputs, List datHeaders, bool skip, Logger logger) + public void DiffCascade(string outDir, bool inplace, List inputs, List datHeaders, bool skip, int maxDegreeOfParallelism, Logger logger) { string post = ""; @@ -395,7 +398,7 @@ namespace SabreTools.Helper.Dats } // Try to output the file - outDats[j].WriteToFile(path, logger); + outDats[j].WriteToFile(path, maxDegreeOfParallelism, logger); } logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); } @@ -406,8 +409,9 @@ namespace SabreTools.Helper.Dats /// Output directory to write the DATs to /// List of inputs to write out from /// Dat headers used optionally + /// Integer representing the maximum amount of parallelization to be used /// Logging object for console and file output - public void MergeNoDiff(string outDir, List inputs, List datHeaders, Logger logger) + public void MergeNoDiff(string outDir, List inputs, List datHeaders, int maxDegreeOfParallelism, Logger logger) { // If we're in SuperDAT mode, prefix all games with their respective DATs if (Type == "SuperDAT") @@ -434,7 +438,7 @@ namespace SabreTools.Helper.Dats } // Try to output the file - WriteToFile(outDir, logger); + WriteToFile(outDir, maxDegreeOfParallelism, logger); } /// @@ -548,11 +552,11 @@ namespace SabreTools.Helper.Dats DatFile innerDatdata = new DatFile(this); logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\""); innerDatdata.Parse(inputFileName, 0, 0, filter, splitType, trim, single, - root, logger, true, clean, descAsName, + root, maxDegreeOfParallelism, logger, true, clean, descAsName, keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); // Try to output the file - innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), logger, overwrite: (outDir != "")); + innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), maxDegreeOfParallelism, logger, overwrite: (outDir != "")); } else if (Directory.Exists(inputFileName)) { @@ -565,11 +569,12 @@ namespace SabreTools.Helper.Dats logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\""); DatFile innerDatdata = new DatFile(this); innerDatdata.Parse(file, 0, 0, filter, splitType, - trim, single, root, logger, true, clean, descAsName, + trim, single, root, maxDegreeOfParallelism, logger, true, clean, descAsName, keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0)); // Try to output the file - innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), logger, overwrite: (outDir != "")); + innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), + maxDegreeOfParallelism, logger, overwrite: (outDir != "")); }); } else diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs index 07a5d394..0072d2cf 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs @@ -38,7 +38,7 @@ namespace SabreTools.Helper.Dats public void Parse(string filename, int sysid, int srcid, Logger logger, bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false) { - Parse(filename, sysid, srcid, new Filter(), SplitType.None, false, false, "", logger, + Parse(filename, sysid, srcid, new Filter(), SplitType.None, false, false, "", 4, logger, keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags); } @@ -54,6 +54,7 @@ namespace SabreTools.Helper.Dats /// True if all games should be replaced by '!', false otherwise /// String representing root directory to compare against for length calculation /// Logger object for console and/or file output + /// Integer representing the maximum amount of parallelization to be used /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if descriptions should be used as names, false otherwise (default) @@ -75,6 +76,7 @@ namespace SabreTools.Helper.Dats string root, // Miscellaneous + int maxDegreeOfParallelism, Logger logger, bool keep = false, bool clean = false, @@ -181,16 +183,16 @@ namespace SabreTools.Helper.Dats switch (splitType) { case SplitType.FullNonMerged: - CreateFullyNonMergedSets(false, logger); + CreateFullyNonMergedSets(false, maxDegreeOfParallelism, logger); break; case SplitType.NonMerged: - CreateNonMergedSets(false, logger); + CreateNonMergedSets(false, maxDegreeOfParallelism, logger); break; case SplitType.Merged: - CreateMergedSets(false, logger); + CreateMergedSets(false, maxDegreeOfParallelism, logger); break; case SplitType.Split: - CreateSplitSets(false, logger); + CreateSplitSets(false, maxDegreeOfParallelism, logger); break; } } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs index 9728e50e..42af1330 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs @@ -145,7 +145,7 @@ namespace SabreTools.Helper.Dats } // Now that we have a list of depots, we want to sort the input DAT by SHA-1 - BucketBy(SortedBy.SHA1, false /* mergeroms */, logger); + BucketBy(SortedBy.SHA1, false /* mergeroms */, maxDegreeOfParallelism, logger); // Then we want to loop through each of the hashes and see if we can rebuild List hashes = Keys.ToList(); @@ -189,7 +189,8 @@ namespace SabreTools.Helper.Dats } // Otherwise, we rebuild that file to all locations that we need to - RebuildIndividualFile(fileinfo, foundpath, outDir, tempDir, date, inverse, outputFormat, romba, updateDat, true /*isZip*/, headerToCheckAgainst, logger); + RebuildIndividualFile(fileinfo, foundpath, outDir, tempDir, date, inverse, outputFormat, romba, + updateDat, true /*isZip*/, headerToCheckAgainst, maxDegreeOfParallelism, logger); } logger.User("Rebuilding complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -202,7 +203,7 @@ namespace SabreTools.Helper.Dats _fileName = "fixDAT_" + _fileName; _name = "fixDAT_" + _name; _description = "fixDAT_" + _description; - WriteToFile(outDir, logger); + WriteToFile(outDir, maxDegreeOfParallelism, logger); } return success; @@ -347,7 +348,7 @@ namespace SabreTools.Helper.Dats _fileName = "fixDAT_" + _fileName; _name = "fixDAT_" + _name; _description = "fixDAT_" + _description; - WriteToFile(outDir, logger); + WriteToFile(outDir, maxDegreeOfParallelism, logger); } return success; @@ -395,7 +396,7 @@ namespace SabreTools.Helper.Dats // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually Rom rom = FileTools.GetFileInfo(file, logger, omitFromScan: (quickScan ? Hash.SecureHashes : Hash.DeepHashes), header: headerToCheckAgainst); usedExternally = RebuildIndividualFile(rom, file, outDir, tempSubDir, date, inverse, outputFormat, - romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + romba, updateDat, false /* isZip */, headerToCheckAgainst, maxDegreeOfParallelism, logger); } // If we're supposed to scan the file internally @@ -410,7 +411,7 @@ namespace SabreTools.Helper.Dats foreach (Rom rom in extracted) { usedInternally &= RebuildIndividualFile(rom, file, outDir, tempSubDir, date, inverse, outputFormat, - romba, updateDat, true /* isZip */, headerToCheckAgainst, logger); + romba, updateDat, true /* isZip */, headerToCheckAgainst, maxDegreeOfParallelism, logger); } } // Otherwise, attempt to extract the files to the temporary directory @@ -430,7 +431,7 @@ namespace SabreTools.Helper.Dats // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually Rom rom = FileTools.GetFileInfo(entry, logger, omitFromScan: (quickScan ? Hash.SecureHashes : Hash.DeepHashes)); usedInternally &= RebuildIndividualFile(rom, entry, outDir, tempSubDir, date, inverse, outputFormat, - romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + romba, updateDat, false /* isZip */, headerToCheckAgainst, maxDegreeOfParallelism, logger); } } // Otherwise, just get the info on the file itself @@ -439,7 +440,7 @@ namespace SabreTools.Helper.Dats // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually Rom rom = FileTools.GetFileInfo(file, logger, omitFromScan: (quickScan ? Hash.SecureHashes : Hash.DeepHashes)); usedExternally = RebuildIndividualFile(rom, file, outDir, tempSubDir, date, inverse, outputFormat, - romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + romba, updateDat, false /* isZip */, headerToCheckAgainst, maxDegreeOfParallelism, logger); } } } @@ -481,25 +482,27 @@ namespace SabreTools.Helper.Dats /// True if the updated DAT should be output, false otherwise /// True if the input file is an archive, false otherwise /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output /// True if the file was able to be rebuilt, false otherwise /// /// TODO: If going from a TGZ file to a TGZ file, don't extract, just copy /// private bool RebuildIndividualFile(Rom rom, string file, string outDir, string tempDir, bool date, - bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool isZip, string headerToCheckAgainst, Logger logger) + bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool isZip, string headerToCheckAgainst, + int maxDegreeOfParallelism, Logger logger) { // Set the output value bool rebuilt = false; // Find if the file has duplicates in the DAT - bool hasDuplicates = rom.HasDuplicates(this, logger); + bool hasDuplicates = rom.HasDuplicates(this, maxDegreeOfParallelism, logger); // If it has duplicates and we're not filtering, rebuild it if (hasDuplicates && !inverse) { // Get the list of duplicates to rebuild to - List dupes = rom.GetDuplicates(this, logger, remove: updateDat); + List dupes = rom.GetDuplicates(this, maxDegreeOfParallelism, logger, remove: updateDat); // If we don't have any duplicates, continue if (dupes.Count == 0) @@ -672,13 +675,13 @@ namespace SabreTools.Helper.Dats Rom headerless = FileTools.GetFileInfo(file + ".new", logger); // Find if the file has duplicates in the DAT - hasDuplicates = headerless.HasDuplicates(this, logger); + hasDuplicates = headerless.HasDuplicates(this, maxDegreeOfParallelism, logger); // If it has duplicates and we're not filtering, rebuild it if (hasDuplicates && !inverse) { // Get the list of duplicates to rebuild to - List dupes = headerless.GetDuplicates(this, logger, remove: updateDat); + List dupes = headerless.GetDuplicates(this, maxDegreeOfParallelism, logger, remove: updateDat); // If we don't have any duplicates, continue if (dupes.Count == 0) @@ -786,9 +789,10 @@ namespace SabreTools.Helper.Dats /// True if only hashes should be checked, false for full file information /// True to enable external scanning of archives, false otherwise /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output /// True if verification was a success, false otherwise - public bool VerifyDepot(List inputs, string tempDir, string headerToCheckAgainst, Logger logger) + public bool VerifyDepot(List inputs, string tempDir, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { // Check the temp directory if (String.IsNullOrEmpty(tempDir)) @@ -830,7 +834,7 @@ namespace SabreTools.Helper.Dats } // Now that we have a list of depots, we want to sort the input DAT by SHA-1 - BucketBy(SortedBy.SHA1, false /* mergeroms */, logger); + BucketBy(SortedBy.SHA1, false /* mergeroms */, maxDegreeOfParallelism, logger); // Then we want to loop through each of the hashes and see if we can rebuild List hashes = Keys.ToList(); @@ -874,7 +878,7 @@ namespace SabreTools.Helper.Dats } // Now we want to remove all duplicates from the DAT - fileinfo.GetDuplicates(this, logger, remove: true); + fileinfo.GetDuplicates(this, maxDegreeOfParallelism, logger, remove: true); } logger.User("Verifying complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); @@ -883,7 +887,7 @@ namespace SabreTools.Helper.Dats _fileName = "fixDAT_" + _fileName; _name = "fixDAT_" + _name; _description = "fixDAT_" + _description; - WriteToFile(null, logger); + WriteToFile(null, maxDegreeOfParallelism, logger); return success; } @@ -896,9 +900,11 @@ namespace SabreTools.Helper.Dats /// True if only hashes should be checked, false for full file information /// True to enable external scanning of archives, false otherwise /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Integer representing the maximum amount of parallelization to be used /// Logger object for file and console output /// True if verification was a success, false otherwise - public bool VerifyGeneric(List inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger) + public bool VerifyGeneric(List inputs, string tempDir, bool hashOnly, bool quickScan, + string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { // Check the temp directory exists if (String.IsNullOrEmpty(tempDir)) @@ -941,7 +947,7 @@ namespace SabreTools.Helper.Dats if (hashOnly) { // First we need to sort by hash to get duplicates - BucketBy(SortedBy.SHA1, false /* mergeroms */, logger); + BucketBy(SortedBy.SHA1, false /* mergeroms */, maxDegreeOfParallelism, logger); // Then follow the same tactics as before foreach (string key in Keys) @@ -977,7 +983,7 @@ namespace SabreTools.Helper.Dats } // Now output the fixdat to the main folder - success &= matched.WriteToFile("", logger, stats: true); + success &= matched.WriteToFile("", maxDegreeOfParallelism, logger, stats: true); return success; } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs index aa7b2ddf..2279f495 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs @@ -29,6 +29,7 @@ namespace SabreTools.Helper.Dats /// /// All information for creating the datfile header /// Set the output directory + /// Integer representing the maximum amount of parallelization to be used /// Logger object for console and/or file output /// True if games should only be compared on game and file name (default), false if system and source are counted /// True if DAT statistics should be output on write, false otherwise (default) @@ -39,7 +40,8 @@ namespace SabreTools.Helper.Dats /// The following features have been requested for file output: /// - Have the ability to strip special (non-ASCII) characters from rom information /// - public bool WriteToFile(string outDir, Logger logger, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) + public bool WriteToFile(string outDir, int maxDegreeOfParallelism, Logger logger, + bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) { // If there's nothing there, abort if (Count == 0) @@ -109,7 +111,7 @@ namespace SabreTools.Helper.Dats } // Bucket roms by game name and optionally dedupe - BucketBy(SortedBy.Game, MergeRoms, logger, norename: norename); + BucketBy(SortedBy.Game, MergeRoms, maxDegreeOfParallelism, logger, norename: norename); // Output the number of items we're going to be writing logger.User("A total of " + Count + " items will be written out to file"); diff --git a/SabreTools.Helper/Tools/ArchiveTools.cs b/SabreTools.Helper/Tools/ArchiveTools.cs index 4470e5e3..295e28be 100644 --- a/SabreTools.Helper/Tools/ArchiveTools.cs +++ b/SabreTools.Helper/Tools/ArchiveTools.cs @@ -1130,10 +1130,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTAR(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTAR(inputFiles, outDir, roms, logger, date: date); } @@ -1316,10 +1314,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTorrent7Zip(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTorrent7Zip(inputFiles, outDir, roms, logger, date: date); } @@ -1378,9 +1374,11 @@ namespace SabreTools.Helper.Tools // If the archive doesn't exist, create it and put the single file if (!File.Exists(archiveFileName)) { - zipFile = new SevenZipCompressor(); - zipFile.ArchiveFormat = OutArchiveFormat.SevenZip; - zipFile.CompressionLevel = SevenZip.CompressionLevel.Normal; + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = SevenZip.CompressionLevel.Normal, + }; // Map all inputs to index Dictionary inputIndexMap = new Dictionary(); @@ -1452,9 +1450,11 @@ namespace SabreTools.Helper.Tools } // Otherwise, process the old zipfile - zipFile = new SevenZipCompressor(); - zipFile.ArchiveFormat = OutArchiveFormat.SevenZip; - zipFile.CompressionLevel = SevenZip.CompressionLevel.Normal; + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = SevenZip.CompressionLevel.Normal, + }; Stream zipFileStream = File.OpenWrite(tempFile); // Get the order for the entries with the new file @@ -1640,10 +1640,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTorrentLRZ(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTorrentLRZ(inputFiles, outDir, roms, logger, date: date); } @@ -1674,10 +1672,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTorrentRAR(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTorrentRAR(inputFiles, outDir, roms, logger, date: date); } @@ -1708,10 +1704,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTorrentXZ(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTorrentXZ(inputFiles, outDir, roms, logger, date: date); } @@ -1742,10 +1736,8 @@ namespace SabreTools.Helper.Tools public static bool WriteTorrentZip(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists - List inputFiles = new List(); - inputFiles.Add(inputFile); - List roms = new List(); - roms.Add(rom); + List inputFiles = new List() { inputFile }; + List roms = new List() { rom }; return WriteTorrentZip(inputFiles, outDir, roms, logger, date: date); }