From 8f3b8a3362a84224dd4135b034e866566cfb4fdd Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Tue, 6 Sep 2016 17:41:44 -0700 Subject: [PATCH] [DATFromDirParallel] Add limit to parallelism --- SabreTools.Helper/Data/Build.cs | 1 + .../Objects/DATFromDirParallel.cs | 19 ++++++++++++++----- SabreTools/Partials/SabreTools_Inits.cs | 8 +++++--- SabreTools/SabreTools.cs | 8 +++++++- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index 6c966187..5fb19b99 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -144,6 +144,7 @@ namespace SabreTools.Helper helptext.Add(" -au=, --author= Set the author of the DAT"); helptext.Add(" -sd, --superdat Enable SuperDAT creation"); helptext.Add(" -t=, --temp= Set the temporary directory to use"); + helptext.Add(" -mt={-1} Amount of threads to use (-1 unlimted)"); helptext.Add(" -es, --ext-split Split a DAT by two file extensions"); helptext.Add(" -exta= First set of extensions (comma-separated)"); helptext.Add(" -extb= Second set of extensions (comma-separated)"); diff --git a/SabreTools.Helper/Objects/DATFromDirParallel.cs b/SabreTools.Helper/Objects/DATFromDirParallel.cs index 8721d24e..340f4dcb 100644 --- a/SabreTools.Helper/Objects/DATFromDirParallel.cs +++ b/SabreTools.Helper/Objects/DATFromDirParallel.cs @@ -26,6 +26,7 @@ namespace SabreTools private bool _archivesAsFiles; private bool _enableGzip; private bool _addblanks; + private int _maxParallelism; // Other required variables private Logger _logger; @@ -47,9 +48,10 @@ namespace SabreTools /// True if archives should be treated as files, false otherwise /// True if GZIP archives should be treated as files, false otherwise /// Name of the directory to create a temp folder in (blank is current directory) - /// True if the file should not be written out, false otherwise (default) + /// Integer representing the maximum amount of parallelization to be used /// Logger object for console and file output - public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger) + public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare, + bool archivesAsFiles, bool enableGzip, string tempDir, int maxParallelism, Logger logger) { _basePath = Path.GetFullPath(basePath); _datdata = datdata; @@ -61,6 +63,7 @@ namespace SabreTools _enableGzip = enableGzip; _addblanks = true; _tempDir = tempDir; + _maxParallelism = maxParallelism; _logger = logger; } @@ -93,7 +96,9 @@ namespace SabreTools _logger.Log("Folder found: " + _basePath); // Process the files in all subfolders - Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories), item => + Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories), + new ParallelOptions { MaxDegreeOfParallelism = 4 }, + item => { ProcessPossibleArchive(item); }); @@ -101,7 +106,9 @@ namespace SabreTools // Now find all folders that are empty, if we are supposed to if (_addblanks) { - Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories), dir => + Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories), + new ParallelOptions { MaxDegreeOfParallelism = 4 }, + dir => { if (Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly).Count() == 0) { @@ -239,7 +246,9 @@ namespace SabreTools if (!encounteredErrors) { _logger.Log(Path.GetFileName(item) + " treated like an archive"); - Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories), entry => + Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories), + new ParallelOptions { MaxDegreeOfParallelism = 4 }, + entry => { ProcessFile(entry, tempSubDir, Path.GetFileNameWithoutExtension(item), _datdata); }); diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index edf4b9e9..abbc7b70 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -424,7 +424,7 @@ namespace SabreTools /// /// Wrap creating a DAT file from files or a directory in parallel /// - /// List of innput filenames + /// List of input filenames /// New filename /// New name /// New description @@ -441,6 +441,7 @@ namespace SabreTools /// True if archives should be treated as files, false otherwise /// True if GZIP archives should be treated as files, false otherwise /// Name of the directory to create a temp folder in (blank is current directory + /// Integer representing the maximum amount of parallelization to be used private static void InitDatFromDirParallel(List inputs, string filename, string name, @@ -457,7 +458,8 @@ namespace SabreTools bool bare, bool archivesAsFiles, bool enableGzip, - string tempDir) + string tempDir, + int maxParallelism) { // Create a new DATFromDir object and process the inputs Dat datdata = new Dat @@ -482,7 +484,7 @@ namespace SabreTools if (Directory.Exists(path)) { string basePath = Path.GetFullPath(path); - DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, _logger); + DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, maxParallelism, _logger); bool success = dfd.Start(); // For DFDParallel only diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index 82b4fe1e..a6c5b8bf 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -126,6 +126,7 @@ namespace SabreTools nodump = null, tsv = null; DiffMode diff = 0x0; + int maxParallelism = -1; long sgt = -1, slt = -1, seq = -1; @@ -502,6 +503,10 @@ namespace SabreTools { md5 = temparg.Split('=')[1]; } + else if (temparg.StartsWith("-mt=") || temparg.StartsWith("--mt=")) + { + Int32.TryParse(temparg.Split('=')[1], out maxParallelism); + } else if (temparg.StartsWith("-n=") || temparg.StartsWith("--name=")) { name = temparg.Split('=')[1]; @@ -740,7 +745,8 @@ namespace SabreTools // Create a DAT from a directory or set of directories in parallel else if (datfromdirparallel) { - InitDatFromDirParallel(inputs, filename, name, description, category, version, author, forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir); + InitDatFromDirParallel(inputs, filename, name, description, category, version, author, + forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir, maxParallelism); } // If we want to run Offline merging mode