[Globals, DatFile, Traverse] Seriously fix parallelization again

SabreTools.Library/Data/Globals.cs

@@ -15,7 +15,7 @@ namespace SabreTools.Library.Data
 		#region Private implementations
 
 		private static Logger _logger = null;
-		private static int _maxDegreeOfParallelism = 4;
+		private static int _maxDegreeOfParallelism = System.Environment.ProcessorCount;
 		private static string _exeName = new Uri(Assembly.GetExecutingAssembly().GetName().CodeBase).LocalPath;
 		private static string _exeDir = Path.GetDirectoryName(_exeName);
 		private static string _args = string.Join(" ", Environment.GetCommandLineArgs());
@@ -39,11 +39,7 @@ namespace SabreTools.Library.Data
 		public static int MaxThreads
 		{
 			get { return _maxDegreeOfParallelism; }
-			set
-			{
-				_maxDegreeOfParallelism = value;
-				System.Threading.ThreadPool.SetMaxThreads(_maxDegreeOfParallelism, _maxDegreeOfParallelism);
-			}
+			set { _maxDegreeOfParallelism = value; }
 		}
 		public static ParallelOptions ParallelOptions
 		{

SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs

@@ -112,7 +112,7 @@ namespace SabreTools.Library.Dats
 			Globals.Logger.User("Processing individual DATs");
 
 			// Parse all of the DATs into their own DatFiles in the array
-			Parallel.For(0, inputs.Count, i =>
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
 			{
 				string input = inputs[i];
 				Globals.Logger.User("Adding DAT: {0}", input.Split('¬')[0]);
@@ -128,21 +128,18 @@ namespace SabreTools.Library.Dats
 			Globals.Logger.User("Processing complete in {0}", DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
 
 			Globals.Logger.User("Populating internal DAT");
-			Parallel.For(0, inputs.Count, i =>
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
 			{
 				// Get the list of keys from the DAT
 				List<string> keys = datHeaders[i].Keys.ToList();
-				Parallel.ForEach(keys, key =>
+				foreach (string key in keys)
 				{
 					// Add everything from the key to the internal DAT
 					AddRange(key, datHeaders[i][key]);
 
 					// Now remove the key from the source DAT
-					lock (datHeaders)
-					{
 					datHeaders[i].Remove(key);
 				}
-				});
 
 				// Now remove the file dictionary from the souce DAT to save memory
 				datHeaders[i].Delete();
@@ -179,7 +176,7 @@ namespace SabreTools.Library.Dats
 			Globals.Logger.User("Populating base DAT for comparison...");
 
 			List<string> baseFileNames = FileTools.GetOnlyFilesFromInputs(basePaths);
-			Parallel.ForEach(baseFileNames, path =>
+			Parallel.ForEach(baseFileNames, Globals.ParallelOptions, path =>
 			{
 				Parse(path, 0, 0, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
 			});
@@ -191,7 +188,7 @@ namespace SabreTools.Library.Dats
 
 			// Now we want to compare each input DAT against the base
 			List<string> inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true);
-			Parallel.ForEach(inputFileNames, path =>
+			foreach (string path in inputFileNames)
 			{
 				// Get the two halves of the path
 				string[] splitpath = path.Split('¬');
@@ -207,7 +204,7 @@ namespace SabreTools.Library.Dats
 
 				// Then we do a hashwise comparison against the base DAT
 				List<string> keys = intDat.Keys.ToList();
-				Parallel.ForEach(keys, key =>
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 				{
 					List<DatItem> datItems = intDat[key];
 					List<DatItem> keepDatItems = new List<DatItem>();
@@ -247,7 +244,7 @@ namespace SabreTools.Library.Dats
 
 				// Due to possible memory requirements, we force a garbage collection
 				GC.Collect();
-			});
+			}
 		}
 
 		/// <summary>
@@ -271,7 +268,7 @@ namespace SabreTools.Library.Dats
 
 			DatFile[] outDatsArray = new DatFile[inputs.Count];
 
-			Parallel.For(0, inputs.Count, j =>
+			Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
 			{
 				string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
 				DatFile diffData;
@@ -301,7 +298,7 @@ namespace SabreTools.Library.Dats
 			Globals.Logger.User("Populating all output DATs");
 			List<string> keys = Keys.ToList();
 
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = DatItem.Merge(this[key]);
 
@@ -330,7 +327,7 @@ namespace SabreTools.Library.Dats
 			start = DateTime.Now;
 			Globals.Logger.User("Outputting all created DATs");
 
-			Parallel.For((skip ? 1 : 0), inputs.Count, j =>
+			Parallel.For((skip ? 1 : 0), inputs.Count, Globals.ParallelOptions, j =>
 			{
 				// If we have an output directory set, replace the path
 				string path = "";
@@ -413,7 +410,7 @@ namespace SabreTools.Library.Dats
 			{
 				DatFile[] outDatsArray = new DatFile[inputs.Count];
 
-				Parallel.For(0, inputs.Count, j =>
+				Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
 				{
 					string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
 					DatFile diffData = new DatFile(this);
@@ -433,7 +430,7 @@ namespace SabreTools.Library.Dats
 			Globals.Logger.User("Populating all output DATs");
 
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = DatItem.Merge(this[key]);
 
@@ -503,7 +500,7 @@ namespace SabreTools.Library.Dats
 			// Output the individual (a-b) DATs
 			if ((diff & DiffMode.Individuals) != 0)
 			{
-				Parallel.For(0, inputs.Count, j =>
+				Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
 				{
 					// If we have an output directory set, replace the path
 					string[] split = inputs[j].Split('¬');
@@ -531,7 +528,7 @@ namespace SabreTools.Library.Dats
 			if (Type == "SuperDAT")
 			{
 				List<string> keys = Keys.ToList();
-				Parallel.ForEach(keys, key =>
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 				{
 					List<DatItem> items = this[key].ToList();
 					List<DatItem> newItems = new List<DatItem>();
@@ -580,8 +577,11 @@ namespace SabreTools.Library.Dats
 		public void Update(List<string> inputFileNames, string outDir, bool inplace, bool clean, bool remUnicode, bool descAsName,
 			Filter filter, SplitType splitType, bool trim, bool single, string root)
 		{
-			Parallel.ForEach(inputFileNames, inputFileName =>
+			for (int i = 0; i < inputFileNames.Count; i++)
 			{
+				// Get the input file name
+				string inputFileName = inputFileNames[i];
+
 				// Clean the input string
 				if (inputFileName != "")
 				{
@@ -618,7 +618,7 @@ namespace SabreTools.Library.Dats
 					}
 
 					List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList();
-					Parallel.ForEach(subFiles, file =>
+					Parallel.ForEach(subFiles, Globals.ParallelOptions, file =>
 					{
 						Globals.Logger.User("Processing '{0}'", Path.GetFullPath(file).Remove(0, inputFileName.Length));
 						DatFile innerDatdata = new DatFile(this);

SabreTools.Library/Dats/Partials/DatFile.DFD.cs

@@ -66,7 +66,7 @@ namespace SabreTools.Library.Dats
 
 				// Process the files in the main folder
 				List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
-				Parallel.ForEach(files, item =>
+				Parallel.ForEach(files, Globals.ParallelOptions, item =>
 				{
 					PopulateFromDirCheckFile(item, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
 						addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
@@ -74,21 +74,21 @@ namespace SabreTools.Library.Dats
 
 				// Find all top-level subfolders
 				files = Directory.EnumerateDirectories(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
-				Parallel.ForEach(files, item =>
+				foreach (string item in files)
 				{
 					List<string> subfiles = Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories).ToList();
-					Parallel.ForEach(subfiles, subitem =>
+					Parallel.ForEach(subfiles, Globals.ParallelOptions, subitem =>
 					{
 						PopulateFromDirCheckFile(subitem, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
 							addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
 					});
-				});
+				}
 
 				// Now find all folders that are empty, if we are supposed to
 				if (!Romba && addBlanks)
 				{
 					List<string> empties = FileTools.GetEmptyDirectories(basePath).ToList();
-					Parallel.ForEach(empties, dir =>
+					Parallel.ForEach(empties, Globals.ParallelOptions, dir =>
 					{
 						// Get the full path for the directory
 						string fulldir = Path.GetFullPath(dir);
@@ -243,7 +243,7 @@ namespace SabreTools.Library.Dats
 			else
 			{
 				// First take care of the found items
-				Parallel.ForEach(extracted, rom =>
+				Parallel.ForEach(extracted, Globals.ParallelOptions, rom =>
 				{
 					PopulateFromDirProcessFileHelper(newItem,
 						rom,
@@ -255,7 +255,7 @@ namespace SabreTools.Library.Dats
 				if (addBlanks)
 				{
 					List<string> empties = ArchiveTools.GetEmptyFoldersInArchive(newItem);
-					Parallel.ForEach(empties, empty =>
+					Parallel.ForEach(empties, Globals.ParallelOptions, empty =>
 					{
 						Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan);
 						PopulateFromDirProcessFileHelper(newItem,

SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs

@@ -52,7 +52,7 @@ namespace SabreTools.Library.Dats
 
 			// First do the initial sort of all of the roms
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> roms = this[key];
 
@@ -143,7 +143,7 @@ namespace SabreTools.Library.Dats
 
 			// Now go through and sort all of the individual lists
 			keys = sortable.Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				// Get the possibly unsorted list
 				List<DatItem> sortedlist = sortable[key];
@@ -242,7 +242,7 @@ namespace SabreTools.Library.Dats
 				// First we want to get a mapping for all games to description
 				ConcurrentDictionary<string, string> mapping = new ConcurrentDictionary<string, string>();
 				List<string> keys = Keys.ToList();
-				Parallel.ForEach(keys, key =>
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 				{
 					List<DatItem> items = this[key];
 					foreach (DatItem item in items)
@@ -257,7 +257,7 @@ namespace SabreTools.Library.Dats
 
 				// Now we loop through every item and update accordingly
 				keys = Keys.ToList();
-				Parallel.ForEach(keys, key =>
+				Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 				{
 					List<DatItem> items = this[key];
 					List<DatItem> newItems = new List<DatItem>();
@@ -312,7 +312,7 @@ namespace SabreTools.Library.Dats
 
 			// Now process all of the roms
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = this[key];
 				for (int j = 0; j < items.Count; j++)

SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs

@@ -128,7 +128,7 @@ namespace SabreTools.Library.Dats
 
 			// Now loop through and get only directories from the input paths
 			List<string> directories = new List<string>();
-			Parallel.ForEach(inputs, input =>
+			Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
 			{
 				// Add to the list if the input is a directory
 				if (Directory.Exists(input))

SabreTools.Library/Dats/Partials/DatFile.Splitters.cs

@@ -85,7 +85,7 @@ namespace SabreTools.Library.Dats
 
 			// Now separate the roms accordingly
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = this[key];
 				foreach (DatItem item in items)
@@ -307,7 +307,7 @@ namespace SabreTools.Library.Dats
 
 			// Now populate each of the DAT objects in turn
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = this[key];
 				foreach (DatItem item in items)
@@ -418,7 +418,7 @@ namespace SabreTools.Library.Dats
 			keys.Sort(SplitByLevelSort);
 
 			// Then, we loop over the games
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				// Here, the key is the name of the game to be used for comparison
 				if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key))
@@ -584,7 +584,7 @@ namespace SabreTools.Library.Dats
 
 			// Now populate each of the DAT objects in turn
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = this[key];
 				foreach(DatItem item in items)

SabreTools.Library/Dats/Partials/DatFile.Statistics.cs

@@ -177,7 +177,7 @@ namespace SabreTools.Library.Dats
 
 			// Loop through and add
 			List<string> keys = Keys.ToList();
-			Parallel.ForEach(keys, key =>
+			Parallel.ForEach(keys, Globals.ParallelOptions, key =>
 			{
 				List<DatItem> items = this[key];
 				foreach(DatItem item in items)
@@ -387,7 +387,7 @@ namespace SabreTools.Library.Dats
 
 			// Make sure we have all files
 			List<Tuple<string, string>> newinputs = new List<Tuple<string, string>>(); // item, basepath
-			Parallel.ForEach(inputs, input =>
+			Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
 			{
 				if (File.Exists(input))
 				{

SabreTools.Library/Dats/Partials/DatFile.Writers.cs

@@ -140,7 +140,7 @@ namespace SabreTools.Library.Dats
 				keys.Sort(new NaturalComparer());
 
 				// Write out all required formats
-				Parallel.ForEach(outfiles.Keys, datFormat =>
+				Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat =>
 				{
 					string outfile = outfiles[datFormat];
 

SabreTools.Library/External/Traverse.cs

@@ -38,7 +38,7 @@ namespace SabreTools.Library.External
 				}
 				subdirs.Clear();
 
-				Parallel.ForEach(dirs, currentDir =>
+				foreach (string currentDir in dirs)
 				{
 					string[] subDirs = Directory.GetDirectories(currentDir);
 
@@ -54,13 +54,14 @@ namespace SabreTools.Library.External
 					try
 					{
 						FileInfo[] files = dir.GetFiles("*.*", SearchOption.TopDirectoryOnly);
-						Parallel.ForEach(files, info =>
+						Parallel.ForEach(files, Globals.ParallelOptions, info =>
 						{
 							action(info);
 						});
 					}
 					catch { }
-				});
+				}
+
 				dirs.Clear();
 			}
 		}