diff --git a/SabreTools.Library/Data/Globals.cs b/SabreTools.Library/Data/Globals.cs index 85000c9b..2134549b 100644 --- a/SabreTools.Library/Data/Globals.cs +++ b/SabreTools.Library/Data/Globals.cs @@ -15,7 +15,7 @@ namespace SabreTools.Library.Data #region Private implementations private static Logger _logger = null; - private static int _maxDegreeOfParallelism = 4; + private static int _maxDegreeOfParallelism = System.Environment.ProcessorCount; private static string _exeName = new Uri(Assembly.GetExecutingAssembly().GetName().CodeBase).LocalPath; private static string _exeDir = Path.GetDirectoryName(_exeName); private static string _args = string.Join(" ", Environment.GetCommandLineArgs()); @@ -39,11 +39,7 @@ namespace SabreTools.Library.Data public static int MaxThreads { get { return _maxDegreeOfParallelism; } - set - { - _maxDegreeOfParallelism = value; - System.Threading.ThreadPool.SetMaxThreads(_maxDegreeOfParallelism, _maxDegreeOfParallelism); - } + set { _maxDegreeOfParallelism = value; } } public static ParallelOptions ParallelOptions { diff --git a/SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs b/SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs index 73e830a5..09607942 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs @@ -112,7 +112,7 @@ namespace SabreTools.Library.Dats Globals.Logger.User("Processing individual DATs"); // Parse all of the DATs into their own DatFiles in the array - Parallel.For(0, inputs.Count, i => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, i => { string input = inputs[i]; Globals.Logger.User("Adding DAT: {0}", input.Split('¬')[0]); @@ -128,21 +128,18 @@ namespace SabreTools.Library.Dats Globals.Logger.User("Processing complete in {0}", DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); Globals.Logger.User("Populating internal DAT"); - Parallel.For(0, inputs.Count, i => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, i => { // Get the list of keys from the DAT List keys = datHeaders[i].Keys.ToList(); - Parallel.ForEach(keys, key => + foreach (string key in keys) { // Add everything from the key to the internal DAT AddRange(key, datHeaders[i][key]); // Now remove the key from the source DAT - lock (datHeaders) - { - datHeaders[i].Remove(key); - } - }); + datHeaders[i].Remove(key); + } // Now remove the file dictionary from the souce DAT to save memory datHeaders[i].Delete(); @@ -179,7 +176,7 @@ namespace SabreTools.Library.Dats Globals.Logger.User("Populating base DAT for comparison..."); List baseFileNames = FileTools.GetOnlyFilesFromInputs(basePaths); - Parallel.ForEach(baseFileNames, path => + Parallel.ForEach(baseFileNames, Globals.ParallelOptions, path => { Parse(path, 0, 0, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName); }); @@ -191,7 +188,7 @@ namespace SabreTools.Library.Dats // Now we want to compare each input DAT against the base List inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true); - Parallel.ForEach(inputFileNames, path => + foreach (string path in inputFileNames) { // Get the two halves of the path string[] splitpath = path.Split('¬'); @@ -207,7 +204,7 @@ namespace SabreTools.Library.Dats // Then we do a hashwise comparison against the base DAT List keys = intDat.Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List datItems = intDat[key]; List keepDatItems = new List(); @@ -247,7 +244,7 @@ namespace SabreTools.Library.Dats // Due to possible memory requirements, we force a garbage collection GC.Collect(); - }); + } } /// @@ -271,7 +268,7 @@ namespace SabreTools.Library.Dats DatFile[] outDatsArray = new DatFile[inputs.Count]; - Parallel.For(0, inputs.Count, j => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => { string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)"; DatFile diffData; @@ -301,7 +298,7 @@ namespace SabreTools.Library.Dats Globals.Logger.User("Populating all output DATs"); List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = DatItem.Merge(this[key]); @@ -330,7 +327,7 @@ namespace SabreTools.Library.Dats start = DateTime.Now; Globals.Logger.User("Outputting all created DATs"); - Parallel.For((skip ? 1 : 0), inputs.Count, j => + Parallel.For((skip ? 1 : 0), inputs.Count, Globals.ParallelOptions, j => { // If we have an output directory set, replace the path string path = ""; @@ -413,7 +410,7 @@ namespace SabreTools.Library.Dats { DatFile[] outDatsArray = new DatFile[inputs.Count]; - Parallel.For(0, inputs.Count, j => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => { string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)"; DatFile diffData = new DatFile(this); @@ -433,7 +430,7 @@ namespace SabreTools.Library.Dats Globals.Logger.User("Populating all output DATs"); List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = DatItem.Merge(this[key]); @@ -503,7 +500,7 @@ namespace SabreTools.Library.Dats // Output the individual (a-b) DATs if ((diff & DiffMode.Individuals) != 0) { - Parallel.For(0, inputs.Count, j => + Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => { // If we have an output directory set, replace the path string[] split = inputs[j].Split('¬'); @@ -531,7 +528,7 @@ namespace SabreTools.Library.Dats if (Type == "SuperDAT") { List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key].ToList(); List newItems = new List(); @@ -580,8 +577,11 @@ namespace SabreTools.Library.Dats public void Update(List inputFileNames, string outDir, bool inplace, bool clean, bool remUnicode, bool descAsName, Filter filter, SplitType splitType, bool trim, bool single, string root) { - Parallel.ForEach(inputFileNames, inputFileName => + for (int i = 0; i < inputFileNames.Count; i++) { + // Get the input file name + string inputFileName = inputFileNames[i]; + // Clean the input string if (inputFileName != "") { @@ -618,7 +618,7 @@ namespace SabreTools.Library.Dats } List subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList(); - Parallel.ForEach(subFiles, file => + Parallel.ForEach(subFiles, Globals.ParallelOptions, file => { Globals.Logger.User("Processing '{0}'", Path.GetFullPath(file).Remove(0, inputFileName.Length)); DatFile innerDatdata = new DatFile(this); diff --git a/SabreTools.Library/Dats/Partials/DatFile.DFD.cs b/SabreTools.Library/Dats/Partials/DatFile.DFD.cs index 0f38ed31..11200a9d 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.DFD.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.DFD.cs @@ -66,7 +66,7 @@ namespace SabreTools.Library.Dats // Process the files in the main folder List files = Directory.EnumerateFiles(basePath, "*", SearchOption.TopDirectoryOnly).ToList(); - Parallel.ForEach(files, item => + Parallel.ForEach(files, Globals.ParallelOptions, item => { PopulateFromDirCheckFile(item, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType, addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst); @@ -74,21 +74,21 @@ namespace SabreTools.Library.Dats // Find all top-level subfolders files = Directory.EnumerateDirectories(basePath, "*", SearchOption.TopDirectoryOnly).ToList(); - Parallel.ForEach(files, item => + foreach (string item in files) { List subfiles = Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories).ToList(); - Parallel.ForEach(subfiles, subitem => + Parallel.ForEach(subfiles, Globals.ParallelOptions, subitem => { PopulateFromDirCheckFile(subitem, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType, addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst); }); - }); + } // Now find all folders that are empty, if we are supposed to if (!Romba && addBlanks) { List empties = FileTools.GetEmptyDirectories(basePath).ToList(); - Parallel.ForEach(empties, dir => + Parallel.ForEach(empties, Globals.ParallelOptions, dir => { // Get the full path for the directory string fulldir = Path.GetFullPath(dir); @@ -243,7 +243,7 @@ namespace SabreTools.Library.Dats else { // First take care of the found items - Parallel.ForEach(extracted, rom => + Parallel.ForEach(extracted, Globals.ParallelOptions, rom => { PopulateFromDirProcessFileHelper(newItem, rom, @@ -255,7 +255,7 @@ namespace SabreTools.Library.Dats if (addBlanks) { List empties = ArchiveTools.GetEmptyFoldersInArchive(newItem); - Parallel.ForEach(empties, empty => + Parallel.ForEach(empties, Globals.ParallelOptions, empty => { Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan); PopulateFromDirProcessFileHelper(newItem, diff --git a/SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs b/SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs index ccb1b2e1..ab4e232c 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs @@ -52,7 +52,7 @@ namespace SabreTools.Library.Dats // First do the initial sort of all of the roms List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List roms = this[key]; @@ -143,7 +143,7 @@ namespace SabreTools.Library.Dats // Now go through and sort all of the individual lists keys = sortable.Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Get the possibly unsorted list List sortedlist = sortable[key]; @@ -242,7 +242,7 @@ namespace SabreTools.Library.Dats // First we want to get a mapping for all games to description ConcurrentDictionary mapping = new ConcurrentDictionary(); List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach (DatItem item in items) @@ -257,7 +257,7 @@ namespace SabreTools.Library.Dats // Now we loop through every item and update accordingly keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; List newItems = new List(); @@ -312,7 +312,7 @@ namespace SabreTools.Library.Dats // Now process all of the roms List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; for (int j = 0; j < items.Count; j++) diff --git a/SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs b/SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs index 2482b5b6..524af444 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs @@ -128,7 +128,7 @@ namespace SabreTools.Library.Dats // Now loop through and get only directories from the input paths List directories = new List(); - Parallel.ForEach(inputs, input => + Parallel.ForEach(inputs, Globals.ParallelOptions, input => { // Add to the list if the input is a directory if (Directory.Exists(input)) diff --git a/SabreTools.Library/Dats/Partials/DatFile.Splitters.cs b/SabreTools.Library/Dats/Partials/DatFile.Splitters.cs index e5b30504..84b30f8e 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.Splitters.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.Splitters.cs @@ -85,7 +85,7 @@ namespace SabreTools.Library.Dats // Now separate the roms accordingly List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach (DatItem item in items) @@ -307,7 +307,7 @@ namespace SabreTools.Library.Dats // Now populate each of the DAT objects in turn List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach (DatItem item in items) @@ -418,7 +418,7 @@ namespace SabreTools.Library.Dats keys.Sort(SplitByLevelSort); // Then, we loop over the games - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Here, the key is the name of the game to be used for comparison if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key)) @@ -584,7 +584,7 @@ namespace SabreTools.Library.Dats // Now populate each of the DAT objects in turn List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach(DatItem item in items) diff --git a/SabreTools.Library/Dats/Partials/DatFile.Statistics.cs b/SabreTools.Library/Dats/Partials/DatFile.Statistics.cs index 30732a06..1786d495 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.Statistics.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.Statistics.cs @@ -177,7 +177,7 @@ namespace SabreTools.Library.Dats // Loop through and add List keys = Keys.ToList(); - Parallel.ForEach(keys, key => + Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach(DatItem item in items) @@ -387,7 +387,7 @@ namespace SabreTools.Library.Dats // Make sure we have all files List> newinputs = new List>(); // item, basepath - Parallel.ForEach(inputs, input => + Parallel.ForEach(inputs, Globals.ParallelOptions, input => { if (File.Exists(input)) { diff --git a/SabreTools.Library/Dats/Partials/DatFile.Writers.cs b/SabreTools.Library/Dats/Partials/DatFile.Writers.cs index 79da4d98..f5273fdb 100644 --- a/SabreTools.Library/Dats/Partials/DatFile.Writers.cs +++ b/SabreTools.Library/Dats/Partials/DatFile.Writers.cs @@ -140,7 +140,7 @@ namespace SabreTools.Library.Dats keys.Sort(new NaturalComparer()); // Write out all required formats - Parallel.ForEach(outfiles.Keys, datFormat => + Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat => { string outfile = outfiles[datFormat]; diff --git a/SabreTools.Library/External/Traverse.cs b/SabreTools.Library/External/Traverse.cs index f8580974..0f3c145b 100644 --- a/SabreTools.Library/External/Traverse.cs +++ b/SabreTools.Library/External/Traverse.cs @@ -38,7 +38,7 @@ namespace SabreTools.Library.External } subdirs.Clear(); - Parallel.ForEach(dirs, currentDir => + foreach (string currentDir in dirs) { string[] subDirs = Directory.GetDirectories(currentDir); @@ -54,13 +54,14 @@ namespace SabreTools.Library.External try { FileInfo[] files = dir.GetFiles("*.*", SearchOption.TopDirectoryOnly); - Parallel.ForEach(files, info => + Parallel.ForEach(files, Globals.ParallelOptions, info => { action(info); }); } catch { } - }); + } + dirs.Clear(); } }