[DATFromDirParallel] Add limit to parallelism

SabreTools.Helper/Data/Build.cs

@@ -144,6 +144,7 @@ namespace SabreTools.Helper
 					helptext.Add("	-au=, --author=		Set the author of the DAT");
 					helptext.Add("	-sd, --superdat		Enable SuperDAT creation");
 					helptext.Add("	-t=, --temp=		Set the temporary directory to use");
+					helptext.Add("	-mt={-1}			Amount of threads to use (-1 unlimted)");
 					helptext.Add("  -es, --ext-split	Split a DAT by two file extensions");
 					helptext.Add("	-exta=			First set of extensions (comma-separated)");
 					helptext.Add("	-extb=			Second set of extensions (comma-separated)");

SabreTools.Helper/Objects/DATFromDirParallel.cs

@@ -26,6 +26,7 @@ namespace SabreTools
 		private bool _archivesAsFiles;
 		private bool _enableGzip;
 		private bool _addblanks;
+		private int _maxParallelism;
 
 		// Other required variables
 		private Logger _logger;
@@ -47,9 +48,10 @@ namespace SabreTools
 		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
 		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
 		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
-		/// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param>
+		/// <param name="maxParallelism">Integer representing the maximum amount of parallelization to be used</param>
 		/// <param name="logger">Logger object for console and file output</param>
-		public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger)
+		public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare,
+			bool archivesAsFiles, bool enableGzip, string tempDir, int maxParallelism, Logger logger)
 		{
 			_basePath = Path.GetFullPath(basePath);
 			_datdata = datdata;
@@ -61,6 +63,7 @@ namespace SabreTools
 			_enableGzip = enableGzip;
 			_addblanks = true;
 			_tempDir = tempDir;
+			_maxParallelism = maxParallelism;
 			_logger = logger;
 		}
 
@@ -93,7 +96,9 @@ namespace SabreTools
 			_logger.Log("Folder found: " + _basePath);
 
 			// Process the files in all subfolders
-			Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories), item =>
+			Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories),
+				new ParallelOptions { MaxDegreeOfParallelism = 4 },
+				item =>
 			{
 				ProcessPossibleArchive(item);
 			});
@@ -101,7 +106,9 @@ namespace SabreTools
 			// Now find all folders that are empty, if we are supposed to
 			if (_addblanks)
 			{
-				Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories), dir =>
+				Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories),
+					new ParallelOptions { MaxDegreeOfParallelism = 4 },
+					dir =>
 				{
 					if (Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly).Count() == 0)
 					{
@@ -239,7 +246,9 @@ namespace SabreTools
 				if (!encounteredErrors)
 				{
 					_logger.Log(Path.GetFileName(item) + " treated like an archive");
-					Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories), entry =>
+					Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories),
+						new ParallelOptions { MaxDegreeOfParallelism = 4 },
+						entry =>
 					{
 						ProcessFile(entry, tempSubDir, Path.GetFileNameWithoutExtension(item), _datdata);
 					});

SabreTools/Partials/SabreTools_Inits.cs

@@ -424,7 +424,7 @@ namespace SabreTools
 		/// <summary>
 		/// Wrap creating a DAT file from files or a directory in parallel
 		/// </summary>
-		/// <param name="input">List of innput filenames</param>
+		/// <param name="inputs">List of input filenames</param>
 		/// <param name="filename">New filename</param>
 		/// <param name="name">New name</param>
 		/// <param name="description">New description</param>
@@ -441,6 +441,7 @@ namespace SabreTools
 		/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
 		/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
 		/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory</param>
+		/// <param name="maxParallelism">Integer representing the maximum amount of parallelization to be used</param>
 		private static void InitDatFromDirParallel(List<string> inputs,
 			string filename,
 			string name,
@@ -457,7 +458,8 @@ namespace SabreTools
 			bool bare,
 			bool archivesAsFiles,
 			bool enableGzip,
-			string tempDir)
+			string tempDir,
+			int maxParallelism)
 		{
 			// Create a new DATFromDir object and process the inputs
 			Dat datdata = new Dat
@@ -482,7 +484,7 @@ namespace SabreTools
 				if (Directory.Exists(path))
 				{
 					string basePath = Path.GetFullPath(path);
-					DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, _logger);
+					DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, maxParallelism, _logger);
 					bool success = dfd.Start();
 
 					// For DFDParallel only

SabreTools/SabreTools.cs

@@ -126,6 +126,7 @@ namespace SabreTools
 				nodump = null,
 				tsv = null;
 			DiffMode diff = 0x0;
+			int maxParallelism = -1;
 			long sgt = -1,
 				slt = -1,
 				seq = -1;
@@ -502,6 +503,10 @@ namespace SabreTools
 						{
 							md5 = temparg.Split('=')[1];
 						}
+						else if (temparg.StartsWith("-mt=") || temparg.StartsWith("--mt="))
+						{
+							Int32.TryParse(temparg.Split('=')[1], out maxParallelism);
+						}
 						else if (temparg.StartsWith("-n=") || temparg.StartsWith("--name="))
 						{
 							name = temparg.Split('=')[1];
@@ -740,7 +745,8 @@ namespace SabreTools
 			// Create a DAT from a directory or set of directories in parallel
 			else if (datfromdirparallel)
 			{
-				InitDatFromDirParallel(inputs, filename, name, description, category, version, author, forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir);
+				InitDatFromDirParallel(inputs, filename, name, description, category, version, author,
+					forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir, maxParallelism);
 			}
 
 			// If we want to run Offline merging mode